def _install_extension_media_internal(self, ext_class):
"""Install static media for an extension.
Performs any installation necessary for an extension. If the extension
has a modern static/ directory, they will be installed into
:file:`{settings.STATIC_ROOT}/ext/`.
"""
if pkg_resources.resource_exists(ext_class.__module__, 'htdocs'):
# This is an older extension that doesn't use the static file
# support. Log a deprecation notice and then install the files.
logger.error('The %s extension uses the deprecated "htdocs" '
'directory for static files. This is no longer '
'supported. It must be updated to use a "static" '
'directory instead.',
ext_class.info.name)
ext_static_path = ext_class.info.installed_static_path
ext_static_path_exists = os.path.exists(ext_static_path)
if ext_static_path_exists:
# Also get rid of the old static contents.
shutil.rmtree(ext_static_path, ignore_errors=True)
if pkg_resources.resource_exists(ext_class.__module__, 'static'):
extracted_path = \
pkg_resources.resource_filename(ext_class.__module__,
'static')
shutil.copytree(extracted_path, ext_static_path, symlinks=True)
def _iouImageBrowserSlot(self):
"""
Slot to open a file browser and select an IOU image.
"""
path = self.getIOUImage(self)
if not path:
return
self.uiIOUPathLineEdit.clear()
self.uiIOUPathLineEdit.setText(path)
if "l2" in path:
# set the default L2 base initial-config
resource_name = "configs/iou_l2_base_initial-config.txt"
if hasattr(sys, "frozen") and os.path.isfile(resource_name):
self.uiInitialConfigLineEdit.setText(os.path.normpath(resource_name))
elif pkg_resources.resource_exists("gns3", resource_name):
iou_base_config_path = pkg_resources.resource_filename("gns3", resource_name)
self.uiInitialConfigLineEdit.setText(os.path.normpath(iou_base_config_path))
else:
# set the default L3 base initial-config
resource_name = "configs/iou_l3_base_initial-config.txt"
if hasattr(sys, "frozen") and os.path.isfile(resource_name):
self.uiInitialConfigLineEdit.setText(os.path.normpath(resource_name))
elif pkg_resources.resource_exists("gns3", resource_name):
iou_base_config_path = pkg_resources.resource_filename("gns3", resource_name)
self.uiInitialConfigLineEdit.setText(os.path.normpath(iou_base_config_path))
def resolve_main_template():
# Plone 5
main_template = os.path.join(
'browser', 'templates', 'main_template.pt')
if pkg_resources.resource_exists('Products.CMFPlone', main_template):
filename = pkg_resources.resource_filename('Products.CMFPlone',
main_template)
return ViewPageTemplateFile(filename)
# Plone 4 with Sunburst
sunburst_main_template = os.path.join(
'skins', 'sunburst_templates', 'main_template.pt')
if pkg_resources.resource_exists('plonetheme.sunburst',
sunburst_main_template):
filename = pkg_resources.resource_filename('plonetheme.sunburst',
sunburst_main_template)
return ViewPageTemplateFile(filename)
# Fallback
skins_main_template = os.path.join(
'skins', 'plone_templates', 'main_template.pt')
if pkg_resources.resource_exists('Products.CMFPlone', skins_main_template):
filename = pkg_resources.resource_filename('Products.CMFPlone',
skins_main_template)
return ViewPageTemplateFile(filename)
def get_style_filepath(style, resolve_dependencies=True):
"""Get the full path of a style file.
:param style: The name of the style (eg. 'apa')
:param resolve_dependencies: If True, for dependent styles the independent
style is parsed and it's path is returned.
:returns: Filepath of a .csl style file.
"""
independent_style = os.path.join(independent_dir, '{0}.csl'.format(style))
if resource_exists(__name__, independent_style):
return resource_filename(__name__, independent_style)
dependent_style = os.path.join(dependent_dir, '{0}.csl'.format(style))
if resource_exists(__name__, dependent_style):
style_path = resource_filename(__name__, dependent_style)
if resolve_dependencies:
inner_style = _resolve_dependent_style(style_path)
inner_style = os.path.join(independent_dir,
'{0}.csl'.format(inner_style))
if resource_exists(__name__, inner_style):
return resource_filename(__name__, inner_style)
else:
raise StyleNotFoundError(
'The independent style {0} was not found.'
.format(inner_style)
)
return style_path
raise StyleNotFoundError('The style {0} was not found.'.format(style))
def find_packaged_regressor(name):
""" Find location of a regression method packaged with YATSM
See :data:`packaged_regressions` for a list of
available pre-packaged regressors
Args:
name (str): name of packaged regression object
Returns:
str: path to packaged regression method
Raises:
KeyError: raise KeyError if user specifies unknown regressor
IOError: raise IOError if the packaged regressor cannot be found
"""
if name not in packaged_regressions:
raise KeyError('Cannot load unknown packaged regressor %s' % name)
path = pkg_resources.resource_filename(__name__, 'pickles')
logger.debug('Checking data files in %s for packaged regressors' % path)
if not pkg_resources.resource_exists(__name__, 'pickles'):
raise IOError('Cannot find packaged regressors in %s. Did you install '
'YATSM via setuptools?' % path)
resource = os.path.join('pickles', name + '.pkl')
if not pkg_resources.resource_exists(__name__, resource):
raise IOError('Cannot find packaged regression method %s, but package '
'directory exists. Check the contents of %s if possible'
% (resource, path))
return pkg_resources.resource_filename(__name__, resource)
def resource_filename(name, pfx=None):
"""
Attempt to find and return the filename of the resource named by the first argument
in the first location of:
# as name in the current directory
# as name in the `pfx` subdirectory of the current directory if provided
# as name relative to the package
# as pfx/name relative to the package
The last two alternatives is used to locate resources distributed in the package.
This includes certain XSLT and XSD files.
:param name: The string name of a resource
:param pfx: An optional prefix to use in searching for name
"""
if os.path.exists(name):
with open(name) as fd:
return fd.read()
elif pfx and os.path.exists(os.path.join(pfx, name)):
with open(os.path.join(pfx, name)) as fd:
return fd.read()
elif pkg_resources.resource_exists(__name__, name):
return pkg_resources.resource_filename(__name__, name)
elif pfx and pkg_resources.resource_exists(__name__, "%s/%s" % (pfx, name)):
return pkg_resources.resource_filename(__name__, "%s/%s" % (pfx, name))
return None
def findEntryPointPlugins(allPlugins):
# look for plugins enabled via setuptools `entry_points`
for entry_point in iter_entry_points(group='girder.plugin'):
# set defaults
allPlugins[entry_point.name] = {
'name': entry_point.name,
'description': '',
'version': '',
'dependencies': set()
}
configJson = os.path.join('girder', 'plugin.json')
configYml = os.path.join('girder', 'plugin.yml')
data = {}
try:
if pkg_resources.resource_exists(entry_point.name, configJson):
with pkg_resources.resource_stream(
entry_point.name, configJson) as conf:
try:
data = json.load(codecs.getreader('utf8')(conf))
except ValueError:
_recordPluginFailureInfo(
plugin=entry_point.name,
traceback=traceback.format_exc())
logprint.exception(
'ERROR: Plugin "%s": plugin.json is not valid '
'JSON.' % entry_point.name)
elif pkg_resources.resource_exists(entry_point.name, configYml):
with pkg_resources.resource_stream(
entry_point.name, configYml) as conf:
try:
data = yaml.safe_load(conf)
except yaml.YAMLError:
_recordPluginFailureInfo(
plugin=entry_point.name,
traceback=traceback.format_exc())
logprint.exception(
'ERROR: Plugin "%s": plugin.yml is not valid '
'YAML.' % entry_point.name)
except (ImportError, SystemError):
# Fall through and just try to load the entry point below. If
# there is still an error, we'll log it there.
pass
if data == {}:
try:
data = getattr(entry_point.load(), 'config', {})
except (ImportError, SystemError):
# If the plugin failed to load via entrypoint, but is in the
# plugins directory, it may still load. We mark and report the
# failure; if it loads later, the failure will be cleared, but
# the report is still desired.
_recordPluginFailureInfo(
plugin=entry_point.name, traceback=traceback.format_exc())
logprint.exception(
'ERROR: Plugin "%s": could not be loaded by entrypoint.' % entry_point.name)
continue
allPlugins[entry_point.name].update(data)
allPlugins[entry_point.name]['dependencies'] = set(
allPlugins[entry_point.name]['dependencies'])
def getSettings(self):
"""
Returns the settings set in this Wizard.
:return: settings dict
"""
path = self.uiIOUImageLineEdit.text()
if self.uiTypeComboBox.currentText() == "L2 image":
# set the default L2 base initial-config
resource_name = "configs/iou_l2_base_initial-config.txt"
if hasattr(sys, "frozen") and os.path.isfile(resource_name):
initial_config = os.path.normpath(resource_name)
elif pkg_resources.resource_exists("gns3", resource_name):
iou_base_config_path = pkg_resources.resource_filename("gns3", resource_name)
initial_config = os.path.normpath(iou_base_config_path)
default_symbol = ":/symbols/multilayer_switch.normal.svg"
hover_symbol = ":/symbols/multilayer_switch.selected.svg"
category = Node.switches
else:
# set the default L3 base initial-config
resource_name = "configs/iou_l3_base_initial-config.txt"
if hasattr(sys, "frozen") and os.path.isfile(resource_name):
initial_config = os.path.normpath(resource_name)
elif pkg_resources.resource_exists("gns3", resource_name):
iou_base_config_path = pkg_resources.resource_filename("gns3", resource_name)
initial_config = os.path.normpath(iou_base_config_path)
default_symbol = ":/symbols/router.normal.svg"
hover_symbol = ":/symbols/router.selected.svg"
category = Node.routers
if IOU.instance().settings()["use_local_server"] or self.uiLocalRadioButton.isChecked():
server = "local"
elif self.uiRemoteRadioButton.isChecked():
if self.uiLoadBalanceCheckBox.isChecked():
server = next(iter(Servers.instance()))
if not server:
QtGui.QMessageBox.critical(self, "IOU device", "No remote server available!")
return
server = "{}:{}".format(server.host, server.port)
else:
server = self.uiRemoteServersComboBox.currentText()
else: # Cloud is selected
server = "cloud"
settings = {
"name": self.uiNameLineEdit.text(),
"path": path,
"image": os.path.basename(path),
"initial_config": initial_config,
"default_symbol": default_symbol,
"category": category,
"hover_symbol": hover_symbol,
"server": server,
}
return settings
def _iouImageBrowserSlot(self):
"""
Slot to open a file browser and select an IOU image.
"""
#TODO: current directory for IOU image + filter?
path, _ = QtGui.QFileDialog.getOpenFileNameAndFilter(self,
"Select an IOU image",
".",
"All files (*.*);;IOU image (*.bin *.image)",
"IOU image (*.bin *.image)")
if not path:
return
if not os.access(path, os.R_OK):
QtGui.QMessageBox.critical(self, "IOU image", "Cannot read {}".format(path))
return
try:
with open(path, "rb") as f:
# read the first 7 bytes of the file.
elf_header_start = f.read(7)
except OSError as e:
QtGui.QMessageBox.critical(self, "IOU image", "Cannot read ELF magic number: {}".format(e))
return
# file must start with the ELF magic number, be 32-bit, little endian and have an ELF version of 1
# normal IOS image are big endian!
if elf_header_start != b'\x7fELF\x01\x01\x01':
QtGui.QMessageBox.critical(self, "IOU image", "Sorry, this is not a valid IOU image!")
return
if not os.access(path, os.X_OK):
QtGui.QMessageBox.critical(self, "IOU image", "{} is not executable".format(path))
return
self.uiIOUPathLineEdit.clear()
self.uiIOUPathLineEdit.setText(path)
if "l2" in path:
# set the default L2 base startup-config
resource_name = "configs/iou_l2_base_startup-config.txt"
if hasattr(sys, "frozen"):
iou_base_config_path = os.path.join(os.path.dirname(sys.executable), resource_name)
self.uiStartupConfigLineEdit.setText(os.path.relpath(os.path.normpath(iou_base_config_path)))
elif pkg_resources.resource_exists("gns3", resource_name):
iou_base_config_path = pkg_resources.resource_filename("gns3", resource_name)
self.uiStartupConfigLineEdit.setText(os.path.relpath(os.path.normpath(iou_base_config_path)))
else:
# set the default L3 base startup-config
resource_name = "configs/iou_l3_base_startup-config.txt"
if hasattr(sys, "frozen"):
iou_base_config_path = os.path.join(os.path.dirname(sys.executable), resource_name)
self.uiStartupConfigLineEdit.setText(os.path.relpath(os.path.normpath(iou_base_config_path)))
elif pkg_resources.resource_exists("gns3", resource_name):
iou_base_config_path = pkg_resources.resource_filename("gns3", resource_name)
self.uiStartupConfigLineEdit.setText(os.path.relpath(os.path.normpath(iou_base_config_path)))
def __getattr__(self, attr):
file_loc = "data/" + self.directory + "/" + attr + ".json"
dir_loc = "data/" + self.directory + "/" + attr
if resource_exists(__name__, file_loc):
return fetch_resource(file_loc)
elif resource_exists(__name__, dir_loc) and \
resource_isdir(__name__, dir_loc):
return CorpusLoader(self.directory + "/" + attr)
else:
raise AttributeError("no resource named " + attr)
def from_defaults(cls, namespace=None, name=None, *args, **kwargs):
if namespace and name and resource_exists(namespace, name):
_namespace = namespace
_name = name
elif (cls.resource_package and cls.resource_default and resource_exists(cls.resource_package, cls.resource_default)):
_namespace = cls.resource_package
_name = cls.resource_default
else:
log.warn('No class resource attributes and no namespace info, returning empty config')
return cls(*args, **kwargs)
return cls.from_pkg_resource(_namespace, _name, *args, **kwargs)
def __init__(self):
# Check if the databases exist.
if (not resource_exists(__name__, "data/prefixes.db") or
not resource_exists(__name__, "data/words.db")):
raise RuntimeError("can't find databases")
# Generate the shelve objects. The '.db' suffix needs to be removed
# from the filenames.
self._prefix_db = shelve.open(resource_filename(__name__,
"data/prefixes.db")[:-3])
self._word_db = shelve.open(resource_filename(__name__,
"data/words.db")[:-3])
def __init__(self, name):
self.name = "Sopra data from %s" % name
import pkg_resources
respath = 'data/Sopra/'
if pkg_resources.resource_exists(__name__, respath+'%s.all' % name):
raise NotImplementedError, "Multi-file luxpop data not implemented"
elif pkg_resources.resource_exists(__name__, respath+'%s.nk' % name):
sopra_file = pkg_resources.resource_stream(__name__, respath+'%s.nk' % (name))
else:
raise RuntimeError, "Couldn't find Sopra datafile"
SopraFile.__init__(self,sopra_file)
def resolve(self, system_url, public_id, context):
"""
Resolves URIs using the resource API
"""
log.debug("resolve SYSTEM URL' %s' for '%s'" % (system_url, public_id))
path = system_url.split("/")
fn = path[len(path) - 1]
if pkg_resources.resource_exists(__name__, fn):
return self.resolve_file(pkg_resources.resource_stream(__name__, fn), context)
elif pkg_resources.resource_exists(__name__, "schema/%s" % fn):
return self.resolve_file(pkg_resources.resource_stream(__name__, "schema/%s" % fn), context)
else:
raise ValueError("Unable to locate %s" % fn)
def get_library(cls):
if cls.library:
return cls.library
if resource_exists("mcviz.utils.svg.data", "texglyph.cache.bz2"):
cls.library = loads(resource_string("mcviz.utils.svg.data", "texglyph.cache.bz2").decode("bz2"))
elif resource_exists("mcviz.utils.svg.data", "texglyph.cache"):
cls.library = loads(resource_string("mcviz.utils.svg.data", "texglyph.cache"))
else:
cls.make_library()
with file(resource_filename("mcviz.utils.svg", "texglyph.cache"), "w") as f:
f.write(dumps(cls.library, 2))
return cls.library
def loadTestsFromModule(self, module):
"""Return a suite of all tests cases contained in the given module
If the module is a package, load tests from all the modules in it.
If the module has an ``additional_tests`` function, call it and add
the return value to the tests.
"""
tests = []
if module.__name__ != 'setuptools.tests.doctest': # ugh
tests.append(TestLoader.loadTestsFromModule(self, module))
if hasattr(module, "additional_tests"):
tests.append(module.additional_tests())
if hasattr(module, '__path__'):
for file in resource_listdir(module.__name__, ''):
if file.endswith('.py') and file != '__init__.py':
submodule = module.__name__ + '.' + file[:-3]
else:
if resource_exists(module.__name__, file + '/__init__.py'):
submodule = module.__name__+'.'+file
else:
continue
tests.append(self.loadTestsFromName(submodule))
if len(tests) != 1:
return self.suiteClass(tests)
else:
return tests[0] # don't create a nested suite for only one return
def get_icon(icon_id):
"Returns an icon from the system theme or our fallback theme if required"
fallback_path = os.path.join("fallback-theme", icon_id + ".png")
if resource_exists(__name__, fallback_path):
return QtGui.QIcon.fromTheme(icon_id, QtGui.QIcon(resource_filename(__name__, fallback_path)))
else:
return QtGui.QIcon.fromTheme(icon_id)
def make_css(minify=False):
print("building css...")
global _css_file
data = []
loaded_modules = module.get_loaded_modules()
for m in loaded_modules:
if not pkg_resources.resource_exists(m, "css"):
continue
for fname in sorted(pkg_resources.resource_listdir(m, "css")):
if not fname.endswith("css"):
continue
res = pkg_resources.resource_string(m, "css/"+fname).decode("utf-8")
data.append(res)
print(" %d css files loaded"%len(data))
if data:
buf = ("\n".join(data)).encode("utf-8")
m = hashlib.new("md5")
m.update(buf)
h = m.hexdigest()[:8]
if not os.path.exists("static/css"):
os.makedirs("static/css")
open("static/css/netforce-%s.css"%h, "wb").write(buf)
if minify:
if not os.path.exists("static/css/netforce-%s-min.css"%h):
print(" minifying css...")
os.system("yui-compressor --type css static/css/netforce-%s.css > static/css/netforce-%s-min.css" %(h,h))
_css_file="netforce-%s-min.css"%h
else:
_css_file="netforce-%s.css"%h
print(" => static/css/%s" % _css_file)
def get_file(path, mode='rb', source=SOURCE_ANY):
"""
Find and open the file at the given ``path``.
This function first searches in the active profile's directory. If the
file is not found there, and configuration is available, it attempts to use
`pkg_resources <http://packages.python.org/distribute/pkg_resources.html>`_
to locate the file, otherwise it will search the root directory.
If a file cannot be found, return ``None``.
**Note:** ``mode`` is not used when getting a file from ``pkg_resources``.
"""
if os.path.isabs(path):
if os.path.isfile(path):
return open(path, mode)
return None
# Not an absolute path, so process it.
ensure_paths()
if source & SOURCE_PROFILE:
p_path = os.path.join(profile_path, path)
if os.path.isfile(p_path):
return open(p_path, mode)
if source & SOURCE_PKG_RESOURCES:
if (package and pr and pr.resource_exists(package, path) and
not pr.resource_isdir(package, path)):
return pr.resource_stream(package, path)
if source & SOURCE_ROOT:
r_path = os.path.join(root_path, path)
if os.path.isfile(r_path):
return open(r_path, mode)
def __call__(self, context, request):
if self.use_subpath:
path_tuple = request.subpath
else:
path_tuple = traversal_path_info(request.environ['PATH_INFO'])
path = _secure_path(path_tuple)
if path is None:
raise HTTPNotFound('Out of bounds: %s' % request.url)
if self.package_name: # package resource
resource_path ='%s/%s' % (self.docroot.rstrip('/'), path)
if resource_isdir(self.package_name, resource_path):
if not request.path_url.endswith('/'):
self.add_slash_redirect(request)
resource_path = '%s/%s' % (resource_path.rstrip('/'),self.index)
if not resource_exists(self.package_name, resource_path):
raise HTTPNotFound(request.url)
filepath = resource_filename(self.package_name, resource_path)
else: # filesystem file
# os.path.normpath converts / to \ on windows
filepath = normcase(normpath(join(self.norm_docroot, path)))
if isdir(filepath):
if not request.path_url.endswith('/'):
self.add_slash_redirect(request)
filepath = join(filepath, self.index)
if not exists(filepath):
raise HTTPNotFound(request.url)
return FileResponse(filepath, request, self.cache_max_age)
def resource(name, pkg=None):
if pkg is None:
pkg = '%s.resources' % calling_package()
resource = resource_filename(pkg, name)
if not resource_exists(resource):
raise OSError('Resource does not exist %s' % repr(resource))
return resource
def _unpack_resource(self, resource_path, resource_name, resource_executable):
if not pkg_resources.resource_exists(__name__, resource_name):
return
if pkg_resources.resource_isdir(__name__, resource_name):
self.__create_dir(resource_path)
for f in pkg_resources.resource_listdir(__name__, resource_name):
if f == "":
# TODO(beng): Figure out why this happens
continue
# TODO: Handle executable resources in directory
self._unpack_resource(
os.path.join(resource_path, f),
os.path.join(resource_name, f),
False,
)
else:
with closable_named_temporary_file(
prefix=resource_path + os.extsep
) as outf:
outf.write(pkg_resources.resource_string(__name__, resource_name))
if resource_executable and hasattr(os, "fchmod"):
st = os.fstat(outf.fileno())
os.fchmod(outf.fileno(), st.st_mode | stat.S_IXUSR)
outf.close()
shutil.copy(outf.name, resource_path)
def expand_macro(self, formatter, name, args):
req = formatter.req
chrome = Chrome(self.env)
report = ReportModule(self.env)
comma_splitter = re.compile(r'(?<!\\),')
kwargs = {}
for arg in comma_splitter.split(args):
arg = arg.replace(r'\,', ',')
m = re.match(r'\s*[^=]+=', arg)
if m:
kw = arg[:m.end() - 1].strip()
value = arg[m.end():]
if re.match(r'^\$[A-Z]*$', value):
value = req.args.get(value[1:])
kwargs[kw] = value if value!= None else ''
else:
if re.match(r'^\$[A-Z]*$', arg):
arg = req.args.get(arg[1:])
id = int(arg)
req.args = kwargs
req.args['page'] = '1'
template, data, content_type = report._render_view(req, id)
add_stylesheet(req, 'common/css/report.css')
fullpath = ''
if pkg_resources.resource_exists('wikireport', 'WikiReport.html'):
fullpath = pkg_resources.resource_filename('wikireport', 'WikiReport.html')
else:
filepath = os.path.dirname(os.path.abspath( __file__ ))
fullpath = os.path.join(filepath, 'WikiReport.html')
return chrome.render_template(req, fullpath, data, None, fragment=True)
def _unpack_dir(self, resource_dir, dst_dir):
if not pkg_resources.resource_exists(__name__, resource_dir):
raise Exception(
"Cannot unpack directory: {0} doesn't exist in the package".format(
resource_dir
)
)
if not pkg_resources.resource_isdir(__name__, resource_dir):
raise Exception(
"Cannot unpack directory: {0} is not a directory".format(resource_dir)
)
self.__create_dir(dst_dir)
if not os.path.exists(dst_dir):
raise Exception(
"Cannot unpack directory: cannot create directory {0}".format(dst_dir)
)
for resource_file in pkg_resources.resource_listdir(__name__, resource_dir):
resource_path = os.path.join(dst_dir, resource_file)
if os.path.exists(resource_path):
continue
self._unpack_resource(
resource_path, "/".join((resource_dir, resource_file)), False
)
def _load_ttbl(lang):
language, territory = _split_language(lang)
language_resource = 'localedata/%s.ttbl' % language
full_resource = 'localedata/%s_%s.ttbl' % (language, territory)
fallback_resource = 'localedata/%s_%s.ttbl' % (language, language.upper())
candidates = [
full_resource + '.bz2',
full_resource,
language_resource + '.bz2',
language_resource,
fallback_resource + '.bz2',
fallback_resource,
]
for c in candidates:
if pkg_resources.resource_exists(__name__, c):
with pkg_resources.resource_stream(__name__, c) as f:
buf = f.read()
if c.endswith('.bz2'):
buf = bz2.decompress(buf)
return pickle.loads(buf)
raise LanguageNotFoundError('Could not find translation table for %s' %
lang)
def run_after(command):
"""
Copy the free-cad macros/modules in the macro directory.
@param command: setuptools command instance, either an InstallCommand or
DevelopCommand.
"""
req = pkg_resources.Requirement.parse(command.distribution.get_name())
if pkg_resources.resource_exists(req, "freecad"):
# get freecad macro dir
macrodir = None
devnull = open(os.devnull, 'w')
for executable in ["freecadcmd", "FreeCADCmd"]:
try:
macrodir = subprocess.check_output([executable, "--get-config", "UserAppData"], stderr=devnull)
break
except:
pass
if macrodir is None:
print "FreeCAD macro directory not found, please read the installation instructions !"
return
macrodir = macrodir.strip()
print "Using macrodir:", macrodir
# recursively copy files/directories to the freecad macro dir
try:
copy_macros(req, "freecad", macrodir)
except Exception, ex:
print "Error:", ex
print "Failed copying macros, please read the installation instructions !"
def getResource(identifier, pkgname=__name__):
"""
Acquire a readable object for a given package name and identifier.
An IOError will be raised if the resource can not be found.
For example:
mydata = getResource('mypkgdata.jpg').read()
Note that the package name must be fully qualified, if given, such
that it would be found in sys.modules.
In some cases, getResource will return a real file object. In that
case, it may be useful to use its name attribute to get the path
rather than use it as a file-like object. For example, you may
be handing data off to a C API.
"""
if resource_exists(pkgname, identifier):
return resource_stream(pkgname, identifier)
mod = sys.modules[pkgname]
fn = getattr(mod, '__file__', None)
if fn is None:
raise IOError("%s has no __file__!" % repr(mod))
path = os.path.join(os.path.dirname(fn), identifier)
loader = getattr(mod, '__loader__', None)
if loader is not None:
try:
data = loader.get_data(path)
except IOError:
pass
else:
return BytesIO(data)
return open(os.path.normpath(path), 'rb')
def __init__(self):
"""Imports all plugin modules based on platform"""
for modname in pkg_resources.resource_listdir(__name__,'plugins'):
if pkg_resources.resource_exists(__name__, 'plugins/%s/__init__.py' % modname):
plugin = __import__('plugins.%s' % modname, globals(), locals(), ['']).get_plugin()
if plugin.platform == 'All' or plugin.platform == howl.platform:
self[modname] = plugin
def _get_template(self, template_name):
"""Given a template name, attempt to resolve it and return the
contents as a String."""
if pkg_resources.resource_exists(__name__, 'templates/%s'
% template_name):
# loading from our recipe egg
return pkg_resources.resource_string(__name__, 'templates/%s'
% template_name)
else:
# see if the user specified a templates directory
template_dir = self.options.get('template_dir',
os.path.join(self.buildout['buildout']['directory'],
'templates'))
if os.path.exists(os.path.join(template_dir, template_name)):
return file(os.path.join(template_dir, template_name)).read()
# unable to find template, throw an error
logging.getLogger(self.name).error("Template %s does not exist." %
template_name)
raise zc.buildout.UserError(
"The specified template, %s, does not exist" % template_name)
def loadTestsFromModule(self, module):
"""Load unit test (skip 'interop' package).
Hacked from the version in 'setuptools.command.test.ScanningLoader'.
"""
tests = []
tests.append(TestLoader.loadTestsFromModule(self,module))
if hasattr(module, '__path__'):
for file in resource_listdir(module.__name__, ''):
if file == 'interop':
# These tests require installing a bunch of extra
# code: see 'src/soaplib/test/README'.
continue
if file.endswith('.py') and file != '__init__.py':
submodule = module.__name__ + '.' + file[:-3]
else:
if resource_exists(
module.__name__, file + '/__init__.py'
):
submodule = module.__name__ + '.' + file
else:
continue
tests.append(self.loadTestsFromName(submodule))
return self.suiteClass(tests)
def get_currency_cross_information(currency_cross, as_json=False):
"""
This function retrieves fundamental financial information from the specified currency cross. The retrieved
information from the currency cross can be valuable as it is additional information that can be used combined
with OHLC values, so to determine financial insights from the company which holds the specified currency cross.
Args:
currency_cross (:obj:`str`): name of the currency_cross to retrieve recent historical data from.
as_json (:obj:`bool`, optional):
optional argument to determine the format of the output data (:obj:`dict` or :obj:`json`).
Returns:
:obj:`pandas.DataFrame` or :obj:`dict`- currency cross_information:
The resulting :obj:`pandas.DataFrame` contains the information fields retrieved from Investing.com
from the specified currency cross; it can also be returned as a :obj:`dict`, if argument `as_json=True`.
If any of the information fields could not be retrieved, that field/s will be filled with
None values. If the retrieval process succeeded, the resulting :obj:`dict` will look like::
currency_cross_information = {
"1-Year Change": "- 1.61%",
"52 wk Range": "1.0879 - 1.1572",
"Ask": 1.1144,
"Bid": 1.114,
"Currency Cross": "EUR/USD",
"Open": 1.1121,
"Prev. Close": 1.1119,
"Todays Range": "1.1123 - 1.1159"
}
Raises:
ValueError: raised if any of the introduced arguments is not valid or errored.
FileNotFoundError: raised if `currency_crosses.csv` file was not found.
IOError: raised if `currency_crosses.csv` file is empty or errored.
RuntimeError: raised if scraping process failed while running.
ConnectionError: raised if the connection to Investing.com errored (did not return HTTP 200)
"""
if not currency_cross:
raise ValueError("ERR#0052: currency_cross param is mandatory and should be a str.")
if not isinstance(currency_cross, str):
raise ValueError("ERR#0052: currency_cross param is mandatory and should be a str.")
if not isinstance(as_json, bool):
raise ValueError("ERR#0002: as_json argument can just be True or False, bool type.")
resource_package = 'investpy'
resource_path = '/'.join(('resources', 'currency_crosses.csv'))
if pkg_resources.resource_exists(resource_package, resource_path):
crosses = pd.read_csv(pkg_resources.resource_filename(resource_package, resource_path))
else:
raise FileNotFoundError("ERR#0060: currency_crosses file not found or errored.")
if crosses is None:
raise IOError("ERR#0050: currency_crosses not found or unable to retrieve.")
currency_cross = currency_cross.strip()
currency_cross = currency_cross.lower()
if unidecode(currency_cross) not in [unidecode(value.lower()) for value in crosses['name'].tolist()]:
raise RuntimeError("ERR#0054: the introduced currency_cross " + str(currency_cross) + " does not exists.")
name = crosses.loc[(crosses['name'].str.lower() == currency_cross).idxmax(), 'name']
tag = crosses.loc[(crosses['name'].str.lower() == currency_cross).idxmax(), 'tag']
url = "https://www.investing.com/currencies/" + tag
head = {
"User-Agent": random_user_agent(),
"X-Requested-With": "XMLHttpRequest",
"Accept": "text/html",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
}
req = requests.get(url, headers=head)
if req.status_code != 200:
raise ConnectionError("ERR#0015: error " + str(req.status_code) + ", try again later.")
root_ = fromstring(req.text)
path_ = root_.xpath("//div[contains(@class, 'overviewDataTable')]/div")
result = pd.DataFrame(columns=['Currency Cross', 'Prev. Close', 'Bid', 'Todays Range',
'Open', 'Ask', '52 wk Range', '1-Year Change'])
result.at[0, 'Currency Cross'] = name
if path_:
for elements_ in path_:
element = elements_.xpath(".//span[@class='float_lang_base_1']")[0]
title_ = element.text_content()
if title_ == "Day's Range":
title_ = 'Todays Range'
if title_ in result.columns.tolist():
try:
result.at[0, title_] = float(element.getnext().text_content().replace(',', ''))
continue
except:
pass
try:
text = element.getnext().text_content().strip()
result.at[0, title_] = datetime.strptime(text, "%b %d, %Y").strftime("%d/%m/%Y")
continue
except:
pass
try:
value = element.getnext().text_content().strip()
if value.__contains__('K'):
value = float(value.replace('K', '').replace(',', '')) * 1e3
elif value.__contains__('M'):
value = float(value.replace('M', '').replace(',', '')) * 1e6
elif value.__contains__('B'):
value = float(value.replace('B', '').replace(',', '')) * 1e9
elif value.__contains__('T'):
value = float(value.replace('T', '').replace(',', '')) * 1e12
result.at[0, title_] = value
continue
except:
pass
result.replace({'N/A': None}, inplace=True)
if as_json is True:
json_ = result.iloc[0].to_dict()
return json_
elif as_json is False:
return result
else:
raise RuntimeError("ERR#0004: data retrieval error while scraping.")
def get_commodities_overview(group, as_json=False, n_results=100):
"""
This function retrieves an overview containing all the real time data available for the main commodities from
every commodity group (metals, softs, meats, energy and grains), such as the names, symbols, current value, etc.
as indexed in Investing.com. So on, the main usage of this function is to get an overview on the main commodities
from a group, so to get a general view. Note that since this function is retrieving a lot of information at once,
by default just the overview of the Top 100 commodities is being retrieved, but an additional parameter called n_results
can be specified so to retrieve N results. Anyways, note that in commodities case, there are just a few ones available.
Args:
group (:obj:`str`): name of the commodity group to retrieve an overview from.
as_json (:obj:`bool`, optional):
optional argument to determine the format of the output data (:obj:`pandas.DataFrame` or :obj:`json`).
n_results (:obj:`int`, optional): number of results to be displayed on the overview table (0-1000).
Returns:
:obj:`pandas.DataFrame` - commodities_overview:
The resulting :obj:`pandas.DataFrame` contains all the data available in Investing.com of the main commodities
from a commodity group in order to get an overview of it.
If the retrieval process succeeded, the resulting :obj:`pandas.DataFrame` should look like::
country | name | last | last_close | high | low | change | change_percentage | currency
--------|------|------|------------|------|-----|--------|-------------------|----------
xxxxxxx | xxxx | xxxx | xxxxxxxxxx | xxxx | xxx | xxxxxx | xxxxxxxxxxxxxxxxx | xxxxxxxx
Raises:
ValueError: raised if any of the introduced arguments errored.
FileNotFoundError: raised if `commodities.csv` file is missing.
IOError: raised if data could not be retrieved due to file error.
RuntimeError:
raised either if the introduced group does not match any of the listed ones or if no overview results could be
retrieved from Investing.com.
ConnectionError: raised if GET requests does not return 200 status code.
"""
if group is None:
raise ValueError(
"ERR#0090: group can not be None, it should be a str.")
if group is not None and not isinstance(group, str):
raise ValueError(
"ERR#0090: group can not be None, it should be a str.")
if not isinstance(as_json, bool):
raise ValueError(
"ERR#0002: as_json argument can just be True or False, bool type.")
if not isinstance(n_results, int):
raise ValueError(
"ERR#0089: n_results argument should be an integer between 1 and 1000."
)
if 1 > n_results or n_results > 1000:
raise ValueError(
"ERR#0089: n_results argument should be an integer between 1 and 1000."
)
resource_package = 'investpy'
resource_path = '/'.join(('resources', 'commodities.csv'))
if pkg_resources.resource_exists(resource_package, resource_path):
commodities = pd.read_csv(
pkg_resources.resource_filename(resource_package, resource_path))
else:
raise FileNotFoundError(
"ERR#0075: commodities file not found or errored.")
if commodities is None:
raise IOError("ERR#0076: commodities not found or unable to retrieve.")
group = unidecode(group.lower())
if group not in get_commodity_groups():
raise RuntimeError(
'ERR#0091: specified commodity group value is not valid.')
commodities = commodities[commodities['group'] == group]
head = {
"User-Agent": random_user_agent(),
"X-Requested-With": "XMLHttpRequest",
"Accept": "text/html",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
}
url = "https://www.investing.com/commodities/" + group
req = requests.get(url, headers=head)
if req.status_code != 200:
raise ConnectionError("ERR#0015: error " + str(req.status_code) +
", try again later.")
root_ = fromstring(req.text)
table = root_.xpath(".//table[@id='cross_rate_1']/tbody/tr")
results = list()
if len(table) > 0:
for row in table[:n_results]:
id_ = row.get('id').replace('pair_', '')
country_check = row.xpath(".//td[@class='flag']/span")[0].get(
'title').lower()
name = row.xpath(
".//td[contains(@class, 'elp')]/a")[0].text_content().strip()
pid = 'pid-' + id_
last = row.xpath(".//td[@class='" + pid +
"-last']")[0].text_content()
last_close = row.xpath(".//td[@class='" + pid +
"-last_close']")[0].text_content()
high = row.xpath(".//td[@class='" + pid +
"-high']")[0].text_content()
low = row.xpath(".//td[@class='" + pid +
"-low']")[0].text_content()
pc = row.xpath(".//td[contains(@class, '" + pid +
"-pc')]")[0].text_content()
pcp = row.xpath(".//td[contains(@class, '" + pid +
"-pcp')]")[0].text_content()
data = {
"country":
country_check if country_check != '' else None,
"name":
name,
"last":
float(last.replace(',', '')),
"last_close":
float(last_close.replace(',', '')),
"high":
float(high.replace(',', '')),
"low":
float(low.replace(',', '')),
"change":
pc,
"change_percentage":
pcp,
"currency":
commodities.loc[(
(commodities['name'] == name) &
(commodities['country'] == country_check)).idxmax(),
'currency'] if country_check != '' else
commodities.loc[(commodities['name'] == name).idxmax(),
'currency']
}
results.append(data)
else:
raise RuntimeError(
"ERR#0092: no data found while retrieving the overview from Investing.com"
)
df = pd.DataFrame(results)
if as_json:
return json.loads(df.to_json(orient='records'))
else:
return df
def _get_available_locales(self):
return sorted(
locale for locale in translation.get_available_locales()
if resource_exists('trac', 'locale/%s/LC_MESSAGES/messages.mo' %
locale))
def test_resource_exists(module_name, resource_name):
assert pkg_resources.resource_exists(module_name, resource_name) is True
def main(argv=None):
if argv is None:
argv = sys.argv
usage = "usage: %prog [options] [workflow_file]"
parser = optparse.OptionParser(usage=usage)
parser.add_option("--no-discovery",
action="store_true",
help="Don't run widget discovery "
"(use full cache instead)")
parser.add_option("--force-discovery",
action="store_true",
help="Force full widget discovery "
"(invalidate cache)")
parser.add_option("--clear-widget-settings",
action="store_true",
help="Remove stored widget setting")
parser.add_option("--no-welcome",
action="store_true",
help="Don't show welcome dialog.")
parser.add_option("--no-splash",
action="store_true",
help="Don't show splash screen.")
parser.add_option("-l",
"--log-level",
help="Logging level (0, 1, 2, 3, 4)",
type="int",
default=1)
parser.add_option("--style",
help="QStyle to use",
type="str",
default=None)
parser.add_option("--stylesheet",
help="Application level CSS style sheet to use",
type="str",
default="orange.qss")
parser.add_option("--qt",
help="Additional arguments for QApplication",
type="str",
default=None)
(options, args) = parser.parse_args(argv[1:])
levels = [
logging.CRITICAL, logging.ERROR, logging.WARN, logging.INFO,
logging.DEBUG
]
# Fix streams before configuring logging (otherwise it will store
# and write to the old file descriptors)
fix_win_pythonw_std_stream()
# Try to fix fonts on OSX Mavericks
fix_osx_10_9_private_font()
# File handler should always be at least INFO level so we need
# the application root level to be at least at INFO.
root_level = min(levels[options.log_level], logging.INFO)
rootlogger = logging.getLogger(canvas.__name__)
rootlogger.setLevel(root_level)
# Initialize SQL query and execution time logger (in SqlTable)
sql_level = min(levels[options.log_level], logging.INFO)
make_sql_logger(sql_level)
# Standard output stream handler at the requested level
stream_hander = logging.StreamHandler()
stream_hander.setLevel(level=levels[options.log_level])
rootlogger.addHandler(stream_hander)
log.info("Starting 'Orange Canvas' application.")
qt_argv = argv[:1]
if options.style is not None:
qt_argv += ["-style", options.style]
if options.qt is not None:
qt_argv += shlex.split(options.qt)
qt_argv += args
log.debug("Starting CanvasApplicaiton with argv = %r.", qt_argv)
app = CanvasApplication(qt_argv)
# NOTE: config.init() must be called after the QApplication constructor
config.init()
clear_settings_flag = os.path.join(config.widget_settings_dir(),
"DELETE_ON_START")
if options.clear_widget_settings or \
os.path.isfile(clear_settings_flag):
log.info("Clearing widget settings")
shutil.rmtree(config.widget_settings_dir(), ignore_errors=True)
file_handler = logging.FileHandler(filename=os.path.join(
config.log_dir(), "canvas.log"),
mode="w")
file_handler.setLevel(root_level)
rootlogger.addHandler(file_handler)
# intercept any QFileOpenEvent requests until the main window is
# fully initialized.
# NOTE: The QApplication must have the executable ($0) and filename
# arguments passed in argv otherwise the FileOpen events are
# triggered for them (this is done by Cocoa, but QApplicaiton filters
# them out if passed in argv)
open_requests = []
def onrequest(url):
log.info("Received an file open request %s", url)
open_requests.append(url)
app.fileOpenRequest.connect(onrequest)
settings = QSettings()
stylesheet = options.stylesheet
stylesheet_string = None
if stylesheet != "none":
if os.path.isfile(stylesheet):
with open(stylesheet, "r") as f:
stylesheet_string = f.read()
else:
if not os.path.splitext(stylesheet)[1]:
# no extension
stylesheet = os.path.extsep.join([stylesheet, "qss"])
pkg_name = canvas.__name__
resource = "styles/" + stylesheet
if pkg_resources.resource_exists(pkg_name, resource):
stylesheet_string = \
pkg_resources.resource_string(pkg_name, resource).decode()
base = pkg_resources.resource_filename(pkg_name, "styles")
pattern = re.compile(
r"^\s@([a-zA-Z0-9_]+?)\s*:\s*([a-zA-Z0-9_/]+?);\s*$",
flags=re.MULTILINE)
matches = pattern.findall(stylesheet_string)
for prefix, search_path in matches:
QDir.addSearchPath(prefix, os.path.join(base, search_path))
log.info("Adding search path %r for prefix, %r",
search_path, prefix)
stylesheet_string = pattern.sub("", stylesheet_string)
else:
log.info("%r style sheet not found.", stylesheet)
# Add the default canvas_icons search path
dirpath = os.path.abspath(os.path.dirname(canvas.__file__))
QDir.addSearchPath("canvas_icons", os.path.join(dirpath, "icons"))
canvas_window = CanvasMainWindow()
canvas_window.setWindowIcon(config.application_icon())
if stylesheet_string is not None:
canvas_window.setStyleSheet(stylesheet_string)
if not options.force_discovery:
reg_cache = cache.registry_cache()
else:
reg_cache = None
widget_discovery = qt.QtWidgetDiscovery(cached_descriptions=reg_cache)
widget_registry = qt.QtWidgetRegistry()
widget_discovery.found_category.connect(widget_registry.register_category)
widget_discovery.found_widget.connect(widget_registry.register_widget)
want_splash = \
settings.value("startup/show-splash-screen", True, type=bool) and \
not options.no_splash
if want_splash:
pm, rect = config.splash_screen()
splash_screen = SplashScreen(pixmap=pm, textRect=rect)
splash_screen.setFont(QFont("Helvetica", 12))
color = QColor("#FFD39F")
def show_message(message):
splash_screen.showMessage(message, color=color)
widget_discovery.discovery_start.connect(splash_screen.show)
widget_discovery.discovery_process.connect(show_message)
widget_discovery.discovery_finished.connect(splash_screen.hide)
log.info("Running widget discovery process.")
cache_filename = os.path.join(cache_dir(), "widget-registry.pck")
if options.no_discovery:
with open(cache_filename, "rb") as f:
widget_registry = pickle.load(f)
widget_registry = qt.QtWidgetRegistry(widget_registry)
else:
widget_discovery.run(config.widgets_entry_points())
# Store cached descriptions
cache.save_registry_cache(widget_discovery.cached_descriptions)
with open(cache_filename, "wb") as f:
pickle.dump(WidgetRegistry(widget_registry), f)
set_global_registry(widget_registry)
canvas_window.set_widget_registry(widget_registry)
canvas_window.show()
canvas_window.raise_()
want_welcome = \
settings.value("startup/show-welcome-screen", True, type=bool) \
and not options.no_welcome
# Process events to make sure the canvas_window layout has
# a chance to activate (the welcome dialog is modal and will
# block the event queue, plus we need a chance to receive open file
# signals when running without a splash screen)
app.processEvents()
app.fileOpenRequest.connect(canvas_window.open_scheme_file)
if want_welcome and not args and not open_requests:
canvas_window.welcome_dialog()
elif args:
log.info("Loading a scheme from the command line argument %r", args[0])
canvas_window.load_scheme(args[0])
elif open_requests:
log.info("Loading a scheme from an `QFileOpenEvent` for %r",
open_requests[-1])
canvas_window.load_scheme(open_requests[-1].toLocalFile())
# local references prevent destruction
_ = show_survey()
__ = check_for_updates()
# Tee stdout and stderr into Output dock
log_view = canvas_window.log_view()
stdout = TextStream()
stdout.stream.connect(log_view.write)
if sys.stdout:
stdout.stream.connect(sys.stdout.write)
stdout.flushed.connect(sys.stdout.flush)
stderr = TextStream()
error_writer = log_view.formated(color=Qt.red)
stderr.stream.connect(error_writer.write)
if sys.stderr:
stderr.stream.connect(sys.stderr.write)
stderr.flushed.connect(sys.stderr.flush)
log.info("Entering main event loop.")
try:
with patch('sys.excepthook',
ExceptHook(stream=stderr, canvas=canvas_window,
handledException=handle_exception)),\
patch('sys.stderr', stderr),\
patch('sys.stdout', stdout):
status = app.exec_()
except BaseException:
log.error("Error in main event loop.", exc_info=True)
canvas_window.deleteLater()
app.processEvents()
app.flush()
del canvas_window
# Collect any cycles before deleting the QApplication instance
gc.collect()
del app
return status
def get_commodity_historical_data(commodity,
from_date,
to_date,
country=None,
as_json=False,
order='ascending',
interval='Daily'):
"""
This function retrieves historical data from the introduced commodity from Investing.com. So on, the historical data
of the introduced commodity in the specified date range will be retrieved and returned as a :obj:`pandas.DataFrame`
if the parameters are valid and the request to Investing.com succeeds. Note that additionally some optional parameters
can be specified: as_json and order, which let the user decide if the data is going to be returned as a :obj:`json` or not,
and if the historical data is going to be ordered ascending or descending (where the index is the date), respectively.
Args:
commodity (:obj:`str`): name of the commodity to retrieve recent data from.
from_date (:obj:`str`): date formatted as `dd/mm/yyyy`, since when data is going to be retrieved.
to_date (:obj:`str`): date formatted as `dd/mm/yyyy`, until when data is going to be retrieved.
country (:obj:`str`, optional):
name of the country to retrieve the commodity data from (if there is more than one country that
provides data from the same commodity).
as_json (:obj:`bool`, optional):
to determine the format of the output data, either a :obj:`pandas.DataFrame` if False and a :obj:`json` if True.
order (:obj:`str`, optional): to define the order of the retrieved data which can either be ascending or descending.
interval (:obj:`str`, optional):
value to define the historical data interval to retrieve, by default `Daily`, but it can also be `Weekly` or `Monthly`.
Returns:
:obj:`pandas.DataFrame` or :obj:`json`:
The function returns a either a :obj:`pandas.DataFrame` or a :obj:`json` file containing the retrieved
historical data of the specified commodity. So on, the resulting dataframe contains the open, high, low and close
values for the selected commodity on market days and the currency in which those values are presented.
The returned data is case we use default arguments will look like::
Date || Open | High | Low | Close | Volume | Currency
-----||------|------|-----|-------|--------|----------
xxxx || xxxx | xxxx | xxx | xxxxx | xxxxxx | xxxxxxxx
but in case that as_json parameter was defined as True, then the output will be::
{
name: name,
historical: [
{
date: 'dd/mm/yyyy',
open: x,
high: x,
low: x,
close: x,
volume: x,
currency: x
},
...
]
}
Raises:
ValueError: raised whenever any of the introduced arguments is not valid or errored.
IOError: raised if commodities object/file was not found or unable to retrieve.
RuntimeError: raised if the introduced commodity was not found or did not match any of the existing ones.
ConnectionError: raised if connection to Investing.com could not be established.
IndexError: raised if commodity historical data was unavailable or not found in Investing.com.
Examples:
>>> data = investpy.get_historical_data(commodity='gold', from_date='01/01/2018', to_date='01/01/2019')
>>> data.head()
Open High Low Close Volume Currency
Date
2018-01-01 1305.8 1309.7 1304.6 1308.7 0 USD
2018-01-02 1370.5 1370.5 1370.5 1370.5 97 USD
2018-01-03 1372.0 1372.0 1369.0 1374.2 22 USD
2018-01-04 1363.4 1375.6 1362.7 1377.4 13 USD
2018-01-05 1377.8 1377.8 1377.8 1378.4 10 USD
"""
if not commodity:
raise ValueError(
"ERR#0078: commodity parameter is mandatory and must be a valid commodity name."
)
if not isinstance(commodity, str):
raise ValueError(
"ERR#0078: commodity parameter is mandatory and must be a valid commodity name."
)
if country is not None and not isinstance(country, str):
raise ValueError("ERR#0025: specified country value not valid.")
if not isinstance(as_json, bool):
raise ValueError(
"ERR#0002: as_json argument can just be True or False, bool type.")
if order not in ['ascending', 'asc', 'descending', 'desc']:
raise ValueError(
"ERR#0003: order argument can just be ascending (asc) or descending (desc), str type."
)
if not interval:
raise ValueError(
"ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'."
)
if not isinstance(interval, str):
raise ValueError(
"ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'."
)
interval = interval.lower()
if interval not in ['daily', 'weekly', 'monthly']:
raise ValueError(
"ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'."
)
try:
datetime.strptime(from_date, '%d/%m/%Y')
except ValueError:
raise ValueError(
"ERR#0011: incorrect from_date date format, it should be 'dd/mm/yyyy'."
)
try:
datetime.strptime(to_date, '%d/%m/%Y')
except ValueError:
raise ValueError(
"ERR#0012: incorrect to_date format, it should be 'dd/mm/yyyy'.")
start_date = datetime.strptime(from_date, '%d/%m/%Y')
end_date = datetime.strptime(to_date, '%d/%m/%Y')
if start_date >= end_date:
raise ValueError(
"ERR#0032: to_date should be greater than from_date, both formatted as 'dd/mm/yyyy'."
)
date_interval = {
'intervals': [],
}
flag = True
while flag is True:
diff = end_date.year - start_date.year
if diff > 19:
obj = {
'start':
start_date.strftime('%m/%d/%Y'),
'end':
start_date.replace(year=start_date.year +
19).strftime('%m/%d/%Y'),
}
date_interval['intervals'].append(obj)
start_date = start_date.replace(year=start_date.year + 19,
day=start_date.day + 1)
else:
obj = {
'start': start_date.strftime('%m/%d/%Y'),
'end': end_date.strftime('%m/%d/%Y'),
}
date_interval['intervals'].append(obj)
flag = False
interval_limit = len(date_interval['intervals'])
interval_counter = 0
data_flag = False
resource_package = 'investpy'
resource_path = '/'.join(('resources', 'commodities.csv'))
if pkg_resources.resource_exists(resource_package, resource_path):
commodities = pd.read_csv(
pkg_resources.resource_filename(resource_package, resource_path))
else:
raise FileNotFoundError(
"ERR#0075: commodities file not found or errored.")
if commodities is None:
raise IOError("ERR#0076: commodities not found or unable to retrieve.")
commodity = commodity.strip()
commodity = commodity.lower()
if unidecode(commodity) not in [
unidecode(value.lower()) for value in commodities['name'].tolist()
]:
raise RuntimeError("ERR#0079: commodity " + commodity +
" not found, check if it is correct.")
if country is None:
found_commodities = commodities[commodities['name'].str.lower() ==
commodity]
if len(found_commodities) > 1:
msg = "Note that the displayed commodity data can differ depending on the country. " \
"If you want to retrieve " + commodity + " data from either " + \
" or ".join(found_commodities['country'].tolist()) + ", specify the country parameter."
warnings.warn(msg, Warning)
del found_commodities
else:
if unidecode(country.lower()) not in commodities['country'].unique(
).tolist():
raise RuntimeError("ERR#0034: country " + country.lower() +
" not found, check if it is correct.")
commodities = commodities[commodities['country'] == unidecode(
country.lower())]
full_name = commodities.loc[(
commodities['name'].str.lower() == commodity).idxmax(), 'full_name']
id_ = commodities.loc[(
commodities['name'].str.lower() == commodity).idxmax(), 'id']
name = commodities.loc[(
commodities['name'].str.lower() == commodity).idxmax(), 'name']
currency = commodities.loc[(
commodities['name'].str.lower() == commodity).idxmax(), 'currency']
header = full_name + ' Historical Data'
final = list()
for index in range(len(date_interval['intervals'])):
interval_counter += 1
params = {
"curr_id": id_,
"smlID": str(randint(1000000, 99999999)),
"header": header,
"st_date": date_interval['intervals'][index]['start'],
"end_date": date_interval['intervals'][index]['end'],
"interval_sec": interval.capitalize(),
"sort_col": "date",
"sort_ord": "DESC",
"action": "historical_data"
}
head = {
"User-Agent": random_user_agent(),
"X-Requested-With": "XMLHttpRequest",
"Accept": "text/html",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
}
url = "https://www.investing.com/instruments/HistoricalDataAjax"
req = requests.post(url, headers=head, data=params)
if req.status_code != 200:
raise ConnectionError("ERR#0015: error " + str(req.status_code) +
", try again later.")
if not req.text:
continue
root_ = fromstring(req.text)
path_ = root_.xpath(".//table[@id='curr_table']/tbody/tr")
result = list()
if path_:
for elements_ in path_:
if elements_.xpath(
".//td")[0].text_content() == 'No results found':
if interval_counter < interval_limit:
data_flag = False
else:
raise IndexError(
"ERR#0080: commodity information unavailable or not found."
)
else:
data_flag = True
info = []
for nested_ in elements_.xpath(".//td"):
info.append(nested_.get('data-real-value'))
if data_flag is True:
commodity_date = datetime.strptime(
str(
datetime.fromtimestamp(
int(info[0]), tz=pytz.timezone('GMT')).date()),
'%Y-%m-%d')
commodity_close = float(info[1].replace(',', ''))
commodity_open = float(info[2].replace(',', ''))
commodity_high = float(info[3].replace(',', ''))
commodity_low = float(info[4].replace(',', ''))
commodity_volume = int(info[5])
result.insert(
len(result),
Data(commodity_date, commodity_open, commodity_high,
commodity_low, commodity_close, commodity_volume,
currency, None))
if data_flag is True:
if order in ['ascending', 'asc']:
result = result[::-1]
elif order in ['descending', 'desc']:
result = result
if as_json is True:
json_list = [value.commodity_as_json() for value in result]
final.append(json_list)
elif as_json is False:
df = pd.DataFrame.from_records(
[value.commodity_to_dict() for value in result])
df.set_index('Date', inplace=True)
final.append(df)
else:
raise RuntimeError(
"ERR#0004: data retrieval error while scraping.")
if order in ['descending', 'desc']:
final.reverse()
if as_json is True:
json_ = {
'name': name,
'historical':
[value for json_list in final for value in json_list]
}
return json.dumps(json_, sort_keys=False)
elif as_json is False:
return pd.concat(final)
def search_commodities(by, value):
"""
This function searches commodities by the introduced value for the specified field. This means that this function
is going to search if there is a value that matches the introduced one for the specified field which is the
`commodities.csv` column name to search in. Available fields to search commodities are 'name', 'full_name' and 'title'.
Args:
by (:obj:`str`): name of the field to search for, which is the column name which can be: ''name', 'full_name' or 'title'.
value (:obj:`str`): value of the field to search for, which is the value that is going to be searched.
Returns:
:obj:`pandas.DataFrame` - search_result:
The resulting :obj:`pandas.DataFrame` contains the search results from the given query, which is
any match of the specified value in the specified field. If there are no results for the given query,
an error will be raised, but otherwise the resulting :obj:`pandas.DataFrame` will contain all the
available commodities that match the introduced query.
Raises:
ValueError: raised if any of the introduced parameters is not valid or errored.
FileNotFoundError: raised if `commodities.csv` file is missing.
IOError: raised if data could not be retrieved due to file error.
RuntimeError: raised if no results were found for the introduced value in the introduced field.
"""
if not by:
raise ValueError(
'ERR#0006: the introduced field to search is mandatory and should be a str.'
)
if not isinstance(by, str):
raise ValueError(
'ERR#0006: the introduced field to search is mandatory and should be a str.'
)
if not value:
raise ValueError(
'ERR#0017: the introduced value to search is mandatory and should be a str.'
)
if not isinstance(value, str):
raise ValueError(
'ERR#0017: the introduced value to search is mandatory and should be a str.'
)
resource_package = 'investpy'
resource_path = '/'.join(('resources', 'commodities.csv'))
if pkg_resources.resource_exists(resource_package, resource_path):
commodities = pd.read_csv(
pkg_resources.resource_filename(resource_package, resource_path))
else:
raise FileNotFoundError(
"ERR#0075: commodities file not found or errored.")
if commodities is None:
raise IOError("ERR#0076: commodities not found or unable to retrieve.")
commodities.drop(columns=['tag', 'id'], inplace=True)
available_search_fields = commodities.columns.tolist()
if isinstance(by, str) and by not in available_search_fields:
raise ValueError(
'ERR#0026: the introduced field to search can either just be ' +
' or '.join(available_search_fields))
commodities['matches'] = commodities[by].str.contains(value, case=False)
search_result = commodities.loc[commodities['matches'] == True].copy()
if len(search_result) == 0:
raise RuntimeError(
'ERR#0043: no results were found for the introduced ' + str(by) +
'.')
search_result.drop(columns=['matches'], inplace=True)
search_result.reset_index(drop=True, inplace=True)
return search_result
def etfs_as_dict(country=None, columns=None, as_json=False):
"""
This function retrieves all the available etfs and returns a dictionary with the specified columns.
Available columns are: 'id', 'name', 'symbol' and 'tag'
All the available etfs can be found at: https://es.investing.com/etfs/spain-etfs
Returns
-------
:returns a dictionary that contains all the available etf values specified in the columns
"""
if country is not None and not isinstance(country, str):
raise IOError("ERR#0025: specified country value not valid.")
if columns is None:
columns = ['id', 'name', 'symbol', 'tag']
else:
if not isinstance(columns, list):
raise ValueError(
"ERR#0020: specified columns argument is not a list, it can just be list type."
)
if not isinstance(as_json, bool):
raise ValueError(
"ERR#0002: as_json argument can just be True or False, bool type.")
resource_package = __name__
resource_path = '/'.join(('resources', 'etfs', 'etfs.csv'))
if pkg_resources.resource_exists(resource_package, resource_path):
etfs = pd.read_csv(
pkg_resources.resource_filename(resource_package, resource_path))
else:
etfs = retrieve_etfs()
if etfs is None:
raise IOError("ERR#0009: etf list not found or unable to retrieve.")
if not all(column in etfs.columns.tolist() for column in columns):
raise ValueError(
"ERR#0021: specified columns does not exist, available columns are <id, name, symbol, tag>"
)
if country is None:
if as_json:
return json.dumps(etfs[columns].to_dict(orient='records'))
else:
return etfs[columns].to_dict(orient='records')
elif country in retrieve_etf_countries():
if as_json:
return json.dumps(
etfs[etfs['country'] == country][columns].to_dict(
orient='records'))
else:
return etfs[etfs['country'] == country][columns].to_dict(
orient='records')
# --- Useless Functions ---
# def get_etf(country):
# """
# This function retrieves all the available etfs to retrieve data from.
# All the available etfs available can be found at: https://es.investing.com/etfs/spain-etfs
#
# Returns
# -------
# :returns a dictionary containing all the etfs information
# """
#
# if country is None or not isinstance(country, str):
# raise IOError("ERR#0028: specified country value not valid.")
#
# head = {
# "User-Agent": ua.get_random(),
# "X-Requested-With": "XMLHttpRequest",
# "Accept": "text/html",
# "Accept-Encoding": "gzip, deflate, br",
# "Connection": "keep-alive",
# }
#
# resource_package = __name__
# resource_path = '/'.join(('resources', 'etfs', 'etf_markets.csv'))
# if pkg_resources.resource_exists(resource_package, resource_path):
# markets = pd.read_csv(pkg_resources.resource_filename(resource_package, resource_path))
# else:
# raise FileNotFoundError("ERR#0027: etf_markets file not found")
#
# for index, row in markets.iterrows():
# if row['country'] == country.lower():
# url = "https://es.investing.com/etfs/" + row['country'].replace(" ", "-") + "-etfs"
#
# req = requests.get(url, headers=head, timeout=15)
#
# if req.status_code != 200:
# raise ConnectionError("ERR#0015: error " + str(req.status_code) + ", try again later.")
#
# root_ = fromstring(req.text)
# path_ = root_.xpath(".//table[@id='etfs']"
# "/tbody"
# "/tr")
#
# results = list()
#
# if path_:
# for elements_ in path_:
# id_ = elements_.get('id').replace('pair_', '')
# symbol = elements_.xpath(".//td[contains(@class, 'symbol')]")[0].get('title')
#
# nested = elements_.xpath(".//a")[0]
# info = nested.get('href').replace('/etfs/', '')
#
# data = {
# "name": nested.text,
# "symbol": symbol,
# "tag": info,
# "id": id_
# }
#
# results.append(data)
# else:
# raise IOError("ERR#0026: specified country etfs not found or unable to retrieve.")
#
# df = pd.DataFrame(results)
#
# return df
def _module_exists(module_name):
"""Returns whether a nrunner "runner" module exists."""
module_filename = f'{module_name}.py'
mod_path = os.path.join('plugins', 'runners', module_filename)
return pkg_resources.resource_exists('avocado', mod_path)
def get_data_file(*dirs):
path = os.path.join("data", *dirs)
if not pkg_resources.resource_exists(__package__, path):
raise FileNotFoundError("Data file '{}' not found".format(path))
return pkg_resources.resource_filename(__package__, path)
def main(argv=None):
if argv is None:
argv = sys.argv
usage = "usage: %prog [options] [workflow_file]"
parser = optparse.OptionParser(usage=usage)
parser.add_option("--no-discovery",
action="store_true",
help="Don't run widget discovery "
"(use full cache instead)")
parser.add_option("--force-discovery",
action="store_true",
help="Force full widget discovery "
"(invalidate cache)")
parser.add_option("--no-welcome",
action="store_true",
help="Don't show welcome dialog.")
parser.add_option("--no-splash",
action="store_true",
help="Don't show splash screen.")
parser.add_option("-l", "--log-level",
help="Logging level (0, 1, 2, 3, 4)",
type="int", default=1)
parser.add_option("--style",
help="QStyle to use",
type="str", default=None)
parser.add_option("--stylesheet",
help="Application level CSS style sheet to use",
type="str", default=None)
parser.add_option("--qt",
help="Additional arguments for QApplication",
type="str", default=None)
parser.add_option("--config",
help="Configuration namespace",
type="str", default="orangecanvas.example")
# -m canvas orange.widgets
# -m canvas --config orange.widgets
(options, args) = parser.parse_args(argv[1:])
levels = [logging.CRITICAL,
logging.ERROR,
logging.WARN,
logging.INFO,
logging.DEBUG]
# Fix streams before configuring logging (otherwise it will store
# and write to the old file descriptors)
fix_win_pythonw_std_stream()
# Set http_proxy environment variable(s) for some clients
fix_set_proxy_env()
# Try to fix macOS automatic window tabbing (Sierra and later)
fix_macos_nswindow_tabbing()
# File handler should always be at least INFO level so we need
# the application root level to be at least at INFO.
root_level = min(levels[options.log_level], logging.INFO)
rootlogger = logging.getLogger(__package__)
rootlogger.setLevel(root_level)
# Standard output stream handler at the requested level
stream_hander = logging.StreamHandler()
stream_hander.setLevel(level=levels[options.log_level])
rootlogger.addHandler(stream_hander)
if options.config is not None:
try:
cfg = utils.name_lookup(options.config)
except (ImportError, AttributeError):
pass
else:
config.set_default(cfg())
log.info("activating %s", options.config)
log.info("Starting 'Orange Canvas' application.")
qt_argv = argv[:1]
style = options.style
defaultstylesheet = "orange.qss"
fusiontheme = None
if style is not None:
if style.startswith("fusion:"):
qt_argv += ["-style", "fusion"]
_, _, fusiontheme = style.partition(":")
else:
qt_argv += ["-style", style]
if options.qt is not None:
qt_argv += shlex.split(options.qt)
qt_argv += args
if QT_VERSION >= 0x50600:
CanvasApplication.setAttribute(Qt.AA_UseHighDpiPixmaps)
log.debug("Starting CanvasApplicaiton with argv = %r.", qt_argv)
app = CanvasApplication(qt_argv)
if app.style().metaObject().className() == "QFusionStyle":
if fusiontheme == "breeze-dark":
app.setPalette(breeze_dark())
defaultstylesheet = "darkorange.qss"
palette = app.palette()
if style is None and palette.color(QPalette.Window).value() < 127:
log.info("Switching default stylesheet to darkorange")
defaultstylesheet = "darkorange.qss"
# NOTE: config.init() must be called after the QApplication constructor
config.init()
file_handler = logging.FileHandler(
filename=os.path.join(config.log_dir(), "canvas.log"),
mode="w"
)
file_handler.setLevel(root_level)
rootlogger.addHandler(file_handler)
# intercept any QFileOpenEvent requests until the main window is
# fully initialized.
# NOTE: The QApplication must have the executable ($0) and filename
# arguments passed in argv otherwise the FileOpen events are
# triggered for them (this is done by Cocoa, but QApplicaiton filters
# them out if passed in argv)
open_requests = []
def onrequest(url):
log.info("Received an file open request %s", url)
open_requests.append(url)
app.fileOpenRequest.connect(onrequest)
settings = QSettings()
stylesheet = options.stylesheet or defaultstylesheet
stylesheet_string = None
if stylesheet != "none":
if os.path.isfile(stylesheet):
with io.open(stylesheet, "r") as f:
stylesheet_string = f.read()
else:
if not os.path.splitext(stylesheet)[1]:
# no extension
stylesheet = os.path.extsep.join([stylesheet, "qss"])
pkg_name = __package__
resource = "styles/" + stylesheet
if pkg_resources.resource_exists(pkg_name, resource):
stylesheet_string = \
pkg_resources.resource_string(pkg_name, resource).decode("utf-8")
base = pkg_resources.resource_filename(pkg_name, "styles")
pattern = re.compile(
r"^\s@([a-zA-Z0-9_]+?)\s*:\s*([a-zA-Z0-9_/]+?);\s*$",
flags=re.MULTILINE
)
matches = pattern.findall(stylesheet_string)
for prefix, search_path in matches:
QDir.addSearchPath(prefix, os.path.join(base, search_path))
log.info("Adding search path %r for prefix, %r",
search_path, prefix)
stylesheet_string = pattern.sub("", stylesheet_string)
else:
log.info("%r style sheet not found.", stylesheet)
# Add the default canvas_icons search path
dirpath = os.path.abspath(os.path.dirname(__file__))
QDir.addSearchPath("canvas_icons", os.path.join(dirpath, "icons"))
canvas_window = CanvasMainWindow()
canvas_window.setAttribute(Qt.WA_DeleteOnClose)
canvas_window.setWindowIcon(config.application_icon())
if stylesheet_string is not None:
canvas_window.setStyleSheet(stylesheet_string)
if not options.force_discovery:
reg_cache = cache.registry_cache()
else:
reg_cache = None
widget_registry = qt.QtWidgetRegistry()
widget_discovery = config.widget_discovery(
widget_registry, cached_descriptions=reg_cache)
want_splash = \
settings.value("startup/show-splash-screen", True, type=bool) and \
not options.no_splash
if want_splash:
pm, rect = config.splash_screen()
splash_screen = SplashScreen(pixmap=pm, textRect=rect)
splash_screen.setAttribute(Qt.WA_DeleteOnClose)
splash_screen.setFont(QFont("Helvetica", 12))
color = QColor("#FFD39F")
def show_message(message):
splash_screen.showMessage(message, color=color)
widget_registry.category_added.connect(show_message)
show_splash = splash_screen.show
close_splash = splash_screen.close
else:
show_splash = close_splash = lambda: None
log.info("Running widget discovery process.")
cache_filename = os.path.join(config.cache_dir(), "widget-registry.pck")
if options.no_discovery:
with open(cache_filename, "rb") as f:
widget_registry = pickle.load(f)
widget_registry = qt.QtWidgetRegistry(widget_registry)
else:
show_splash()
widget_discovery.run(config.widgets_entry_points())
close_splash()
# Store cached descriptions
cache.save_registry_cache(widget_discovery.cached_descriptions)
with open(cache_filename, "wb") as f:
pickle.dump(WidgetRegistry(widget_registry), f)
set_global_registry(widget_registry)
canvas_window.set_widget_registry(widget_registry)
canvas_window.show()
canvas_window.raise_()
want_welcome = \
settings.value("startup/show-welcome-screen", True, type=bool) \
and not options.no_welcome
# Process events to make sure the canvas_window layout has
# a chance to activate (the welcome dialog is modal and will
# block the event queue, plus we need a chance to receive open file
# signals when running without a splash screen)
app.processEvents()
app.fileOpenRequest.connect(canvas_window.open_scheme_file)
if want_welcome and not args and not open_requests:
canvas_window.welcome_dialog()
elif args:
log.info("Loading a scheme from the command line argument %r",
args[0])
canvas_window.load_scheme(args[0])
elif open_requests:
log.info("Loading a scheme from an `QFileOpenEvent` for %r",
open_requests[-1])
canvas_window.load_scheme(open_requests[-1].toLocalFile())
# Tee stdout and stderr into Output dock
output_view = canvas_window.output_view()
stdout = TextStream()
stdout.stream.connect(output_view.write)
if sys.stdout:
stdout.stream.connect(sys.stdout.write)
stdout.flushed.connect(sys.stdout.flush)
stderr = TextStream()
error_writer = output_view.formated(color=Qt.red)
stderr.stream.connect(error_writer.write)
if sys.stderr:
stderr.stream.connect(sys.stderr.write)
stderr.flushed.connect(sys.stderr.flush)
with ExitStack() as stack:
stack.enter_context(closing(stderr))
stack.enter_context(closing(stdout))
stack.enter_context(redirect_stdout(stdout))
stack.enter_context(redirect_stderr(stderr))
log.info("Entering main event loop.")
sys.excepthook = ExceptHook(stream=stderr)
try:
status = app.exec_()
finally:
sys.excepthook = sys.__excepthook__
del canvas_window
app.processEvents()
app.flush()
# Collect any cycles before deleting the QApplication instance
gc.collect()
del app
return status
def main(args, pass_through_args):
conf = ClusterConfig()
conf.extend_flag('image-version', IMAGE_VERSION)
if not pkg_resources.resource_exists('hailtop.hailctl', "deploy.yaml"):
raise RuntimeError(f"package has no 'deploy.yaml' file")
deploy_metadata = yaml.safe_load(
pkg_resources.resource_stream('hailtop.hailctl',
"deploy.yaml"))['dataproc']
conf.extend_flag('properties', DEFAULT_PROPERTIES)
if args.properties:
conf.parse_and_extend('properties', args.properties)
# default to highmem machines if using VEP
if not args.worker_machine_type:
args.worker_machine_type = 'n1-highmem-8' if args.vep else 'n1-standard-8'
# default initialization script to start up cluster with
conf.extend_flag('initialization-actions',
[deploy_metadata['init_notebook.py']])
# add VEP init script
if args.vep:
conf.extend_flag('initialization-actions',
[deploy_metadata[f'vep-{args.vep}.sh']])
# add custom init scripts
if args.init:
conf.extend_flag('initialization-actions', args.init.split(','))
if args.metadata:
conf.parse_and_extend('metadata', args.metadata)
wheel = args.wheel or deploy_metadata['wheel']
conf.extend_flag('metadata', {'WHEEL': wheel})
# if Python packages requested, add metadata variable
packages = deploy_metadata['pip_dependencies'].strip('|').split('|||')
metadata_pkgs = conf.flags['metadata'].get('PKGS')
split_regex = r'[|,]'
if metadata_pkgs:
packages.extend(re.split(split_regex, metadata_pkgs))
if args.packages:
packages.extend(re.split(split_regex, args.packages))
conf.extend_flag('metadata', {'PKGS': '|'.join(packages)})
def disk_size(size):
if args.vep:
size = max(size, 200)
return str(size) + 'GB'
# rewrite metadata to escape it
conf.flags['metadata'] = '^|||^' + '|||'.join(
f'{k}={v}' for k, v in conf.flags['metadata'].items())
conf.extend_flag(
'properties', {
"spark:spark.driver.memory":
"{driver_memory}g".format(driver_memory=str(
int(MACHINE_MEM[args.master_machine_type] *
args.master_memory_fraction)))
})
conf.flags['master-machine-type'] = args.master_machine_type
conf.flags['master-boot-disk-size'] = '{}GB'.format(
args.master_boot_disk_size)
conf.flags['num-master-local-ssds'] = args.num_master_local_ssds
conf.flags['num-preemptible-workers'] = args.num_preemptible_workers
conf.flags['num-worker-local-ssds'] = args.num_worker_local_ssds
conf.flags['num-workers'] = args.num_workers
conf.flags['preemptible-worker-boot-disk-size'] = disk_size(
args.preemptible_worker_boot_disk_size)
conf.flags['worker-boot-disk-size'] = disk_size(args.worker_boot_disk_size)
conf.flags['worker-machine-type'] = args.worker_machine_type
conf.flags['zone'] = args.zone
conf.flags['initialization-action-timeout'] = args.init_timeout
if args.network:
conf.flags['network'] = args.network
if args.configuration:
conf.flags['configuration'] = args.configuration
if args.project:
conf.flags['project'] = args.project
if args.bucket:
conf.flags['bucket'] = args.bucket
try:
label = sp.check_output(['gcloud', 'config', 'get-value', 'account'])
conf.flags['labels'] = 'creator=' + re.sub(
r'[^0-9a-z_\-]', '_',
label.decode().strip().lower())[:63]
except sp.CalledProcessError as e:
sys.stderr.write(
"Warning: could not run 'gcloud config get-value account': " +
e.output.decode() + "\n")
# command to start cluster
cmd = conf.get_command(args.name)
if args.beta or args.max_idle or args.max_age:
cmd.insert(1, 'beta')
if args.max_idle:
cmd.append('--max-idle={}'.format(args.max_idle))
if args.max_age:
cmd.append('--max-age={}'.format(args.max_age))
cmd.extend(pass_through_args)
# print underlying gcloud command
print(' '.join(cmd[:5]) + ' \\\n ' + ' \\\n '.join(cmd[5:]))
# spin up cluster
if not args.dry_run:
print("Starting cluster '{}'...".format(args.name))
sp.check_call(cmd)
if args.master_tags:
sp.check_call([
'gcloud', 'compute', 'instances', 'add-tags', args.name + '-m',
'--tags', args.master_tags
])
def main(args, pass_through_args):
modify_args = []
if args.num_workers is not None:
modify_args.append('--num-workers={}'.format(args.num_workers))
if args.num_preemptible_workers is not None:
modify_args.append('--num-preemptible-workers={}'.format(args.num_preemptible_workers))
if args.graceful_decommission_timeout:
if not modify_args:
sys.exit("Error: Cannot use --graceful-decommission-timeout without resizing the cluster.")
modify_args.append('--graceful-decommission-timeout={}'.format(args.graceful_decommission_timeout))
if args.max_idle:
modify_args.append('--max-idle={}'.format(args.max_idle))
if args.update_hail_version and args.wheel:
sys.exit("Error: Cannot specify both --update-hail-version and --wheel")
if modify_args:
cmd = ['gcloud',
'dataproc',
'clusters',
'update',
args.name] + modify_args
if args.beta:
cmd.insert(1, 'beta')
cmd.extend(pass_through_args)
# print underlying gcloud command
print(' '.join(cmd[:5]) + ' \\\n ' + ' \\\n '.join(cmd[5:]))
# Update cluster
if not args.dry_run:
print("Updating cluster '{}'...".format(args.name))
check_call(cmd)
wheel = None
if args.update_hail_version:
assert pkg_resources.resource_exists('hailtop.hailctl', "deploy.yaml")
updated_wheel = yaml.safe_load(
pkg_resources.resource_stream('hailtop.hailctl', "deploy.yaml"))['dataproc']['wheel']
wheel = updated_wheel
else:
wheel = args.wheel
if wheel is not None:
wheelfile = os.path.basename(wheel)
cmds = []
if wheel.startswith("gs://"):
cmds.append([
'gcloud',
'compute',
'ssh',
'{}-m'.format(args.name),
'--zone={}'.format(args.zone),
'--',
f'sudo gsutil cp {wheel} /tmp/ && '
'sudo /opt/conda/default/bin/pip uninstall -y hail && '
f'sudo /opt/conda/default/bin/pip install --no-dependencies /tmp/{wheelfile}'
])
else:
cmds.extend([
[
'gcloud',
'compute',
'scp',
'--zone={}'.format(args.zone),
wheel,
'{}-m:/tmp/'.format(args.name)
],
[
'gcloud',
'compute',
'ssh',
f'{args.name}-m',
f'--zone={args.zone}',
'--',
'sudo /opt/conda/default/bin/pip uninstall -y hail && '
f'sudo /opt/conda/default/bin/pip install --no-dependencies /tmp/{wheelfile}'
]
])
for cmd in cmds:
print(cmd)
check_call(cmd)
if not wheel and not modify_args and pass_through_args:
sys.stderr.write('ERROR: found pass-through arguments but not known modification args.')
sys.exit(1)
def get_bond_information(bond, as_json=False):
"""
This function retrieves fundamental financial information from the specified bond. The retrieved
information from the bond can be valuable as it is additional information that can be used combined
with OHLC values, so to determine financial insights from the company which holds the specified bond.
Args:
bond (:obj:`str`): name of the bond to retrieve information from.
as_json (:obj:`bool`, optional):
optional argument to determine the format of the output data (:obj:`dict` or :obj:`json`).
Returns:
:obj:`pandas.DataFrame` or :obj:`dict`- bond_information:
The resulting :obj:`pandas.DataFrame` contains the information fields retrieved from Investing.com
from the specified bond; it can also be returned as a :obj:`dict`, if argument `as_json=True`.
If any of the information fields could not be retrieved, that field/s will be filled with
None values. If the retrieval process succeeded, the resulting :obj:`dict` will look like::
bond_information = {
"1-Year Change": "46.91%",
"52 wk Range": "-0.575 - 0.01",
"Bond Name": "Spain 1Y",
"Coupon": "None",
"Maturity Date": "04/12/2020",
"Prev. Close": -0.425,
"Price": 100.417,
"Price Open": 100.416,
"Price Range": -100.481,
"Todays Range": "-0.49 - -0.424"
}
Raises:
ValueError: raised if any of the introduced arguments is not valid or errored.
FileNotFoundError: raised if bonds.csv file was not found or errored.
IOError: raised if bonds.csv file is empty or errored.
RuntimeError: raised if scraping process failed while running.
ConnectionError: raised if the connection to Investing.com errored (did not return HTTP 200)
"""
if not bond:
raise ValueError(
"ERR#0066: bond parameter is mandatory and must be a valid bond name."
)
if not isinstance(bond, str):
raise ValueError("ERR#0067: bond argument needs to be a str.")
if not isinstance(as_json, bool):
raise ValueError(
"ERR#0002: as_json argument can just be True or False, bool type.")
resource_package = 'investpy'
resource_path = '/'.join(('resources', 'bonds.csv'))
if pkg_resources.resource_exists(resource_package, resource_path):
bonds = pd.read_csv(
pkg_resources.resource_filename(resource_package, resource_path))
else:
raise FileNotFoundError("ERR#0064: bonds file not found or errored.")
if bonds is None:
raise IOError(
"ERR#0065: bonds object not found or unable to retrieve.")
bond = unidecode(bond.strip().lower())
if bond not in list(bonds['name'].str.lower()):
raise RuntimeError("ERR#0068: bond " + bond +
" not found, check if it is correct.")
name = bonds.loc[(bonds['name'].str.lower() == bond).idxmax(), 'name']
tag = bonds.loc[(bonds['name'].str.lower() == bond).idxmax(), 'tag']
url = "https://www.investing.com/rates-bonds/" + tag
head = {
"User-Agent": random_user_agent(),
"X-Requested-With": "XMLHttpRequest",
"Accept": "text/html",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
}
req = requests.get(url, headers=head)
if req.status_code != 200:
raise ConnectionError("ERR#0015: error " + str(req.status_code) +
", try again later.")
root_ = fromstring(req.text)
path_ = root_.xpath("//div[contains(@class, 'overviewDataTable')]/div")
result = pd.DataFrame(columns=[
'Bond Name', 'Prev. Close', 'Price', 'Coupon', 'Todays Range',
'Price Open', 'Maturity Date', '52 wk Range', 'Price Range',
'1-Year Change'
])
result.at[0, 'Bond Name'] = name
if path_:
for elements_ in path_:
element = elements_.xpath(".//span[@class='float_lang_base_1']")[0]
title_ = element.text_content()
if title_ == "Day's Range":
title_ = 'Todays Range'
if title_ in result.columns.tolist():
try:
result.at[0, title_] = float(
element.getnext().text_content().replace(',', ''))
continue
except:
pass
try:
text = element.getnext().text_content().strip()
result.at[0, title_] = datetime.strptime(
text, "%d %b %Y").strftime("%d/%m/%Y")
continue
except:
pass
try:
text = element.getnext().text_content().strip()
occ = text.count('-')
if occ == 1:
reg = re.compile(r"([\-]{1}[ ]{1}[0-9\.]+)+")
matches = reg.findall(text)
if len(matches) > 0:
result.at[0, title_] = float(matches[0].replace(
' ', ''))
continue
elif occ == 2:
reg = re.compile(r"([\-]{1}[ ]{1}[0-9\.]+)+")
matches = reg.findall(text)
if len(matches) > 0:
res = matches[0].replace(' ', '')
result.at[0, title_] = ' '.join([res, matches[1]])
continue
elif occ == 3:
reg = re.compile(r"([\-]{1}[ ]{1}[0-9\.]+)+")
matches = reg.findall(text)
if len(matches) > 0:
res = list()
for match in matches:
res.append(match.replace(' ', ''))
result.at[0, title_] = ' - '.join(res)
continue
except:
pass
try:
value = element.getnext().text_content().strip()
if value.__contains__('K'):
value = float(value.replace('K', '').replace(',',
'')) * 1e3
elif value.__contains__('M'):
value = float(value.replace('M', '').replace(',',
'')) * 1e6
elif value.__contains__('B'):
value = float(value.replace('B', '').replace(',',
'')) * 1e9
elif value.__contains__('T'):
value = float(value.replace('T', '').replace(
',', '')) * 1e12
result.at[0, title_] = value
continue
except:
pass
result.replace({'N/A': None}, inplace=True)
if as_json is True:
json_ = result.iloc[0].to_dict()
return json_
elif as_json is False:
return result
else:
raise RuntimeError("ERR#0004: data retrieval error while scraping.")
def get_currency_cross_historical_data(currency_cross, from_date, to_date, as_json=False, order='ascending', interval='Daily'):
"""
This function retrieves recent historical data from the introduced `currency_cross` from Investing
via Web Scraping. The resulting data can it either be stored in a :obj:`pandas.DataFrame` or in a
:obj:`json` file, with `ascending` or `descending` order.
Args:
currency_cross (:obj:`str`): name of the currency cross to retrieve recent historical data from.
from_date (:obj:`str`): date as `str` formatted as `dd/mm/yyyy`, from where data is going to be retrieved.
to_date (:obj:`str`): date as `str` formatted as `dd/mm/yyyy`, until where data is going to be retrieved.
as_json (:obj:`bool`, optional):
optional argument to determine the format of the output data (:obj:`pandas.DataFrame` or :obj:`json`).
order (:obj:`str`, optional):
optional argument to define the order of the retrieved data (`ascending`, `asc` or `descending`, `desc`).
interval (:obj:`str`, optional):
value to define the historical data interval to retrieve, by default `Daily`, but it can also be `Weekly` or `Monthly`.
Returns:
:obj:`pandas.DataFrame` or :obj:`json`:
The function returns a either a :obj:`pandas.DataFrame` or a :obj:`json` file containing the retrieved
recent data from the specified currency_cross via argument. The dataset contains the open, high, low, close and
volume values for the selected currency_cross on market days.
The return data is case we use default arguments will look like::
Date || Open | High | Low | Close | Currency
-----||------|------|-----|-------|---------
xxxx || xxxx | xxxx | xxx | xxxxx | xxxxxxxx
but if we define `as_json=True`, then the output will be::
{
name: name,
historical: [
dd/mm/yyyy: {
'open': x,
'high': x,
'low': x,
'close': x,
'currency' : x
},
...
]
}
Raises:
ValueError: argument error.
IOError: stocks object/file not found or unable to retrieve.
RuntimeError: introduced currency_cross does not match any of the indexed ones.
ConnectionError: if GET requests does not return 200 status code.
IndexError: if currency_cross information was unavailable or not found.
Examples:
>>> data = investpy.get_currency_cross_historical_data(currency_cross='EUR/USD', from_date='01/01/2018', to_date='01/01/2019')
>>> data.head()
Open High Low Close Currency
Date
2018-01-01 1.2003 1.2014 1.1995 1.2010 USD
2018-01-02 1.2013 1.2084 1.2003 1.2059 USD
2018-01-03 1.2058 1.2070 1.2001 1.2014 USD
2018-01-04 1.2015 1.2090 1.2004 1.2068 USD
2018-01-05 1.2068 1.2085 1.2021 1.2030 USD
"""
if not currency_cross:
raise ValueError("ERR#0052: currency_cross param is mandatory and should be a str.")
if not isinstance(currency_cross, str):
raise ValueError("ERR#0052: currency_cross param is mandatory and should be a str.")
try:
datetime.strptime(from_date, '%d/%m/%Y')
except ValueError:
raise ValueError("ERR#0011: incorrect data format, it should be 'dd/mm/yyyy'.")
try:
datetime.strptime(to_date, '%d/%m/%Y')
except ValueError:
raise ValueError("ERR#0011: incorrect data format, it should be 'dd/mm/yyyy'.")
start_date = datetime.strptime(from_date, '%d/%m/%Y')
end_date = datetime.strptime(to_date, '%d/%m/%Y')
if start_date >= end_date:
raise ValueError("ERR#0032: to_date should be greater than from_date, both formatted as 'dd/mm/yyyy'.")
if not isinstance(as_json, bool):
raise ValueError("ERR#0002: as_json argument can just be True or False, bool type.")
if order not in ['ascending', 'asc', 'descending', 'desc']:
raise ValueError("ERR#0003: order argument can just be ascending (asc) or descending (desc), str type.")
if not interval:
raise ValueError("ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'.")
if not isinstance(interval, str):
raise ValueError("ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'.")
if interval not in ['Daily', 'Weekly', 'Monthly']:
raise ValueError("ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'.")
date_interval = {
'intervals': [],
}
flag = True
while flag is True:
diff = end_date.year - start_date.year
if diff > 19:
obj = {
'start': start_date.strftime('%m/%d/%Y'),
'end': start_date.replace(year=start_date.year + 19).strftime('%m/%d/%Y'),
}
date_interval['intervals'].append(obj)
start_date = start_date.replace(year=start_date.year + 19) + timedelta(days=1)
else:
obj = {
'start': start_date.strftime('%m/%d/%Y'),
'end': end_date.strftime('%m/%d/%Y'),
}
date_interval['intervals'].append(obj)
flag = False
interval_limit = len(date_interval['intervals'])
interval_counter = 0
data_flag = False
resource_package = 'investpy'
resource_path = '/'.join(('resources', 'currency_crosses.csv'))
if pkg_resources.resource_exists(resource_package, resource_path):
currency_crosses = pd.read_csv(pkg_resources.resource_filename(resource_package, resource_path))
else:
raise FileNotFoundError("ERR#0060: currency_crosses file not found or errored.")
if currency_crosses is None:
raise IOError("ERR#0050: currency_crosses not found or unable to retrieve.")
currency_cross = currency_cross.strip()
currency_cross = currency_cross.lower()
if unidecode(currency_cross) not in [unidecode(value.lower()) for value in currency_crosses['name'].tolist()]:
raise RuntimeError("ERR#0054: the introduced currency_cross " + str(currency_cross) + " does not exists.")
id_ = currency_crosses.loc[(currency_crosses['name'].str.lower() == currency_cross).idxmax(), 'id']
name = currency_crosses.loc[(currency_crosses['name'].str.lower() == currency_cross).idxmax(), 'name']
currency = currency_crosses.loc[(currency_crosses['name'].str.lower() == currency_cross).idxmax(), 'second']
final = list()
header = name + ' Historical Data'
for index in range(len(date_interval['intervals'])):
interval_counter += 1
params = {
"curr_id": id_,
"smlID": str(randint(1000000, 99999999)),
"header": header,
"st_date": date_interval['intervals'][index]['start'],
"end_date": date_interval['intervals'][index]['end'],
"interval_sec": interval,
"sort_col": "date",
"sort_ord": "DESC",
"action": "historical_data"
}
head = {
"User-Agent": random_user_agent(),
"X-Requested-With": "XMLHttpRequest",
"Accept": "text/html",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
}
url = "https://www.investing.com/instruments/HistoricalDataAjax"
req = requests.post(url, headers=head, data=params)
if req.status_code != 200:
raise ConnectionError("ERR#0015: error " + str(req.status_code) + ", try again later.")
if not req.text:
continue
root_ = fromstring(req.text)
path_ = root_.xpath(".//table[@id='curr_table']/tbody/tr")
result = list()
if path_:
for elements_ in path_:
info = []
for nested_ in elements_.xpath(".//td"):
info.append(nested_.get('data-real-value'))
if elements_.xpath(".//td")[0].text_content() == 'No results found':
if interval_counter < interval_limit:
data_flag = False
else:
raise IndexError("ERR#0055: currency_cross information unavailable or not found.")
else:
data_flag = True
if data_flag is True:
currency_cross_date = datetime.strptime(str(datetime.fromtimestamp(int(info[0]), tz=pytz.utc).date()), '%Y-%m-%d')
currency_cross_close = float(info[1].replace(',', ''))
currency_cross_open = float(info[2].replace(',', ''))
currency_cross_high = float(info[3].replace(',', ''))
currency_cross_low = float(info[4].replace(',', ''))
result.insert(len(result),
Data(currency_cross_date, currency_cross_open, currency_cross_high, currency_cross_low,
currency_cross_close, None, currency, None))
if data_flag is True:
if order in ['ascending', 'asc']:
result = result[::-1]
elif order in ['descending', 'desc']:
result = result
if as_json is True:
json_ = {
'name': name,
'historical':
[value.currency_cross_as_json() for value in result]
}
final.append(json_)
elif as_json is False:
df = pd.DataFrame.from_records([value.currency_cross_to_dict() for value in result])
df.set_index('Date', inplace=True)
final.append(df)
else:
raise RuntimeError("ERR#0004: data retrieval error while scraping.")
if as_json is True:
return json.dumps(final[0], sort_keys=False)
elif as_json is False:
return pd.concat(final)
def load_manifest_schema():
resource_args = (__name__, '../manifest_schema.yaml')
assert pkg_resources.resource_exists(*resource_args)
yaml_data = pkg_resources.resource_string(*resource_args)
return ruamel.yaml.load(yaml_data)
def get_bonds_overview(country, as_json=False):
"""
This function retrieves an overview containing all the real time data available for the government bonds
from a country, such as the names, symbols, current value, etc. as indexed in Investing.com. So on, the main
usage of this function is to get an overview on the government bonds from a country, so to get a general view.
Args:
country (:obj:`str`): name of the country to retrieve the government bonds overview from.
as_json (:obj:`bool`, optional):
optional argument to determine the format of the output data (:obj:`pandas.DataFrame` or :obj:`json`).
Returns:
:obj:`pandas.DataFrame` - bonds_overview:
The resulting :obj:`pandas.DataFrame` contains all the data available in Investing.com of the government
bonds from a country in order to get an overview of it.
If the retrieval process succeeded, the resulting :obj:`pandas.DataFrame` should look like::
country | name | last | last_close | high | low | change | change_percentage
--------|------|------|------------|------|-----|--------|-------------------
xxxxxxx | xxxx | xxxx | xxxxxxxxxx | xxxx | xxx | xxxxxx | xxxxxxxxxxxxxxxxx
Raises:
ValueError: raised if any of the introduced arguments is not valid or errored.
FileNotFoundError: raised if `bonds.csv` file is missing.
IOError: raised if data could not be retrieved due to file error.
RuntimeError:
raised either if the introduced country does not match any of the listed ones or if no overview results could be
retrieved from Investing.com.
ConnectionError: raised if GET requests does not return 200 status code.
"""
if country is None:
raise ValueError(
"ERR#0039: country can not be None, it should be a str.")
if country is not None and not isinstance(country, str):
raise ValueError("ERR#0025: specified country value not valid.")
if not isinstance(as_json, bool):
raise ValueError(
"ERR#0002: as_json argument can just be True or False, bool type.")
resource_package = 'investpy'
resource_path = '/'.join(('resources', 'bonds.csv'))
if pkg_resources.resource_exists(resource_package, resource_path):
bonds = pd.read_csv(
pkg_resources.resource_filename(resource_package, resource_path))
else:
raise FileNotFoundError("ERR#0064: bonds file not found or errored.")
if bonds is None:
raise IOError(
"ERR#0065: bonds object not found or unable to retrieve.")
country = unidecode(country.strip().lower())
if country not in get_bond_countries():
raise ValueError("ERR#0034: country " + country +
" not found, check if it is correct.")
bonds = bonds[bonds['country'] == country]
if country == 'united states':
country = 'usa'
elif country == 'united kingdom':
country = 'uk'
head = {
"User-Agent": random_user_agent(),
"X-Requested-With": "XMLHttpRequest",
"Accept": "text/html",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
}
url = 'https://www.investing.com/rates-bonds/' + country + '-government-bonds'
req = requests.get(url, headers=head)
if req.status_code != 200:
raise ConnectionError("ERR#0015: error " + str(req.status_code) +
", try again later.")
root_ = fromstring(req.text)
table = root_.xpath(".//table[@id='cr1']/tbody/tr")
results = list()
if len(table) > 0:
for row in table:
id_ = row.get('id').replace('pair_', '')
country_check = row.xpath(".//td[@class='flag']/span")[0].get(
'title').lower()
name = row.xpath(
".//td[contains(@class, 'elp')]/a")[0].text_content().strip()
pid = 'pid-' + id_
last = row.xpath(".//td[@class='" + pid +
"-last']")[0].text_content()
last_close = row.xpath(".//td[@class='" + pid +
"-last_close']")[0].text_content()
high = row.xpath(".//td[@class='" + pid +
"-high']")[0].text_content()
low = row.xpath(".//td[@class='" + pid +
"-low']")[0].text_content()
pc = row.xpath(".//td[contains(@class, '" + pid +
"-pc')]")[0].text_content()
pcp = row.xpath(".//td[contains(@class, '" + pid +
"-pcp')]")[0].text_content()
data = {
"country": country_check,
"name": name,
"last": float(last.replace(',', '')),
"last_close": float(last_close.replace(',', '')),
"high": float(high.replace(',', '')),
"low": float(low.replace(',', '')),
"change": pc,
"change_percentage": pcp
}
results.append(data)
else:
raise RuntimeError(
"ERR#0092: no data found while retrieving the overview from Investing.com"
)
df = pd.DataFrame(results)
if as_json:
return json.loads(df.to_json(orient='records'))
else:
return df
def get_currency_cross_recent_data(currency_cross, as_json=False, order='ascending', interval='Daily'):
"""
This function retrieves recent historical data from the introduced `currency_cross` as indexed in Investing.com
via Web Scraping. The resulting data can it either be stored in a :obj:`pandas.DataFrame` or in a
:obj:`json` file, with `ascending` or `descending` order.
Args:
currency_cross (:obj:`str`): name of the currency_cross to retrieve recent historical data from.
as_json (:obj:`bool`, optional):
optional argument to determine the format of the output data (:obj:`pandas.DataFrame` or :obj:`json`).
order (:obj:`str`, optional):
optional argument to define the order of the retrieved data (`ascending`, `asc` or `descending`, `desc`).
interval (:obj:`str`, optional):
value to define the historical data interval to retrieve, by default `Daily`, but it can also be `Weekly` or `Monthly`.
Returns:
:obj:`pandas.DataFrame` or :obj:`json`:
The function returns a either a :obj:`pandas.DataFrame` or a :obj:`json` file containing the retrieved
recent data from the specified currency_cross via argument. The dataset contains the open, high, low, close,
volume and currency values for the selected currency_cross on market days.
The return data is in case we use default arguments will look like::
Date || Open | High | Low | Close | Currency
-----||------|------|-----|-------|---------
xxxx || xxxx | xxxx | xxx | xxxxx | xxxxxxxx
but if we define `as_json=True`, then the output will be::
{
name: name,
recent: [
dd/mm/yyyy: {
'open': x,
'high': x,
'low': x,
'close': x,
'currency' : x
},
...
]
}
Raises:
ValueError: raised if any of the introduced arguments was not valid or errored.
IOError: raised if currency_crosses object/file not found or unable to retrieve.
RuntimeError: raised introduced currency_cross does not match any of the indexed ones.
ConnectionError: raised if GET request did not return 200 status code.
IndexError: raised if currency_cross information was unavailable or not found.
Examples:
>>> data = investpy.get_currency_cross_recent_data(currency_cross='EUR/USD')
>>> data.head()
Open High Low Close Currency
Date
2019-08-27 1.1101 1.1116 1.1084 1.1091 USD
2019-08-28 1.1090 1.1099 1.1072 1.1078 USD
2019-08-29 1.1078 1.1093 1.1042 1.1057 USD
2019-08-30 1.1058 1.1062 1.0963 1.0991 USD
2019-09-02 1.0990 1.1000 1.0958 1.0968 USD
"""
if not currency_cross:
raise ValueError("ERR#0052: currency_cross param is mandatory and should be a str.")
if not isinstance(currency_cross, str):
raise ValueError("ERR#0052: currency_cross param is mandatory and should be a str.")
if not isinstance(as_json, bool):
raise ValueError("ERR#0002: as_json argument can just be True or False, bool type.")
if order not in ['ascending', 'asc', 'descending', 'desc']:
raise ValueError("ERR#0003: order argument can just be ascending (asc) or descending (desc), str type.")
if not interval:
raise ValueError("ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'.")
if not isinstance(interval, str):
raise ValueError("ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'.")
if interval not in ['Daily', 'Weekly', 'Monthly']:
raise ValueError("ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'.")
resource_package = 'investpy'
resource_path = '/'.join(('resources', 'currency_crosses.csv'))
if pkg_resources.resource_exists(resource_package, resource_path):
currency_crosses = pd.read_csv(pkg_resources.resource_filename(resource_package, resource_path))
else:
raise FileNotFoundError("ERR#0060: currency_crosses file not found or errored.")
if currency_crosses is None:
raise IOError("ERR#0050: currency_crosses not found or unable to retrieve.")
currency_cross = currency_cross.strip()
currency_cross = currency_cross.lower()
if unidecode(currency_cross) not in [unidecode(value.lower()) for value in currency_crosses['name'].tolist()]:
raise RuntimeError("ERR#0054: the introduced currency_cross " + str(currency_cross) + " does not exists.")
id_ = currency_crosses.loc[(currency_crosses['name'].str.lower() == currency_cross).idxmax(), 'id']
name = currency_crosses.loc[(currency_crosses['name'].str.lower() == currency_cross).idxmax(), 'name']
currency = currency_crosses.loc[(currency_crosses['name'].str.lower() == currency_cross).idxmax(), 'second']
header = name + ' Historical Data'
params = {
"curr_id": id_,
"smlID": str(randint(1000000, 99999999)),
"header": header,
"interval_sec": interval,
"sort_col": "date",
"sort_ord": "DESC",
"action": "historical_data"
}
head = {
"User-Agent": random_user_agent(),
"X-Requested-With": "XMLHttpRequest",
"Accept": "text/html",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
}
url = "https://www.investing.com/instruments/HistoricalDataAjax"
req = requests.post(url, headers=head, data=params)
if req.status_code != 200:
raise ConnectionError("ERR#0015: error " + str(req.status_code) + ", try again later.")
root_ = fromstring(req.text)
path_ = root_.xpath(".//table[@id='curr_table']/tbody/tr")
result = list()
if path_:
for elements_ in path_:
if elements_.xpath(".//td")[0].text_content() == 'No results found':
raise IndexError("ERR#0055: currency_cross information unavailable or not found.")
info = []
for nested_ in elements_.xpath(".//td"):
info.append(nested_.get('data-real-value'))
currency_cross_date = datetime.strptime(str(datetime.fromtimestamp(int(info[0]), tz=pytz.utc).date()), '%Y-%m-%d')
currency_cross_close = float(info[1].replace(',', ''))
currency_cross_open = float(info[2].replace(',', ''))
currency_cross_high = float(info[3].replace(',', ''))
currency_cross_low = float(info[4].replace(',', ''))
result.insert(len(result),
Data(currency_cross_date, currency_cross_open, currency_cross_high, currency_cross_low,
currency_cross_close, None, currency, None))
if order in ['ascending', 'asc']:
result = result[::-1]
elif order in ['descending', 'desc']:
result = result
if as_json is True:
json_ = {
'name': name,
'recent':
[value.currency_cross_as_json() for value in result]
}
return json.dumps(json_, sort_keys=False)
elif as_json is False:
df = pd.DataFrame.from_records([value.currency_cross_to_dict() for value in result])
df.set_index('Date', inplace=True)
return df
else:
raise RuntimeError("ERR#0004: data retrieval error while scraping.")
def get_bond_historical_data(bond,
from_date,
to_date,
as_json=False,
order='ascending',
interval='Daily'):
"""
This function retrieves historical data from the introduced bond from Investing.com. So on, the historical data
of the introduced bond in the specified date range will be retrieved and returned as a :obj:`pandas.DataFrame` if
the parameters are valid and the request to Investing.com succeeds. Note that additionally some optional parameters
can be specified: as_json and order, which let the user decide if the data is going to be returned as a :obj:`json`
or not, and if the historical data is going to be ordered ascending or descending (where the index is the date),
respectively.
Args:
bond (:obj:`str`): name of the bond to retrieve historical data from.
from_date (:obj:`str`): date formatted as `dd/mm/yyyy`, since when data is going to be retrieved.
to_date (:obj:`str`): date formatted as `dd/mm/yyyy`, until when data is going to be retrieved.
as_json (:obj:`bool`, optional):
to determine the format of the output data, either a :obj:`pandas.DataFrame` if False and a :obj:`json` if True.
order (:obj:`str`, optional): to define the order of the retrieved data which can either be ascending or descending.
interval (:obj:`str`, optional):
value to define the historical data interval to retrieve, by default `Daily`, but it can also be `Weekly` or `Monthly`.
Returns:
:obj:`pandas.DataFrame` or :obj:`json`:
The function returns a either a :obj:`pandas.DataFrame` or a :obj:`json` file containing the retrieved
recent data from the specified bond via argument. The dataset contains the open, high, low and close for the
selected bond on market days.
The resulting recent data, in case that the default parameters were applied, will look like::
Date || Open | High | Low | Close
-----||------|------|-----|-------
xxxx || xxxx | xxxx | xxx | xxxxx
but in case that as_json parameter was defined as True, then the output will be::
{
name: name,
historical: [
{
date: 'dd/mm/yyyy',
open: x,
high: x,
low: x,
close: x
},
...
]
}
Raises:
ValueError: raised whenever any of the introduced arguments is not valid or errored.
IOError: raised if bonds object/file was not found or unable to retrieve.
RuntimeError: raised if the introduced bond was not found or did not match any of the existing ones.
ConnectionError: raised if connection to Investing.com could not be established.
IndexError: raised if bond historical data was unavailable or not found in Investing.com.
Examples:
>>> data = investpy.get_bond_historical_data(bond='Argentina 3Y', from_date='01/01/2010', to_date='01/01/2019')
>>> data.head()
Open High Low Close
Date
2011-01-03 4.15 4.15 4.15 5.15
2011-01-04 4.07 4.07 4.07 5.45
2011-01-05 4.27 4.27 4.27 5.71
2011-01-10 4.74 4.74 4.74 6.27
2011-01-11 4.30 4.30 4.30 6.56
"""
if not bond:
raise ValueError(
"ERR#0066: bond parameter is mandatory and must be a valid bond name."
)
if not isinstance(bond, str):
raise ValueError("ERR#0067: bond argument needs to be a str.")
if not isinstance(as_json, bool):
raise ValueError(
"ERR#0002: as_json argument can just be True or False, bool type.")
if order not in ['ascending', 'asc', 'descending', 'desc']:
raise ValueError(
"ERR#0003: order argument can just be ascending (asc) or descending (desc), str type."
)
if not interval:
raise ValueError(
"ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'."
)
if not isinstance(interval, str):
raise ValueError(
"ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'."
)
interval = interval.lower()
if interval not in ['daily', 'weekly', 'monthly']:
raise ValueError(
"ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'."
)
try:
datetime.strptime(from_date, '%d/%m/%Y')
except ValueError:
raise ValueError(
"ERR#0011: incorrect from_date date format, it should be 'dd/mm/yyyy'."
)
try:
datetime.strptime(to_date, '%d/%m/%Y')
except ValueError:
raise ValueError(
"ERR#0012: incorrect to_date format, it should be 'dd/mm/yyyy'.")
start_date = datetime.strptime(from_date, '%d/%m/%Y')
end_date = datetime.strptime(to_date, '%d/%m/%Y')
if start_date >= end_date:
raise ValueError(
"ERR#0032: to_date should be greater than from_date, both formatted as 'dd/mm/yyyy'."
)
date_interval = {
'intervals': [],
}
flag = True
while flag is True:
diff = end_date.year - start_date.year
if diff > 19:
obj = {
'start':
start_date.strftime('%m/%d/%Y'),
'end':
start_date.replace(year=start_date.year +
19).strftime('%m/%d/%Y'),
}
date_interval['intervals'].append(obj)
start_date = start_date.replace(year=start_date.year +
19) + timedelta(days=1)
else:
obj = {
'start': start_date.strftime('%m/%d/%Y'),
'end': end_date.strftime('%m/%d/%Y'),
}
date_interval['intervals'].append(obj)
flag = False
interval_limit = len(date_interval['intervals'])
interval_counter = 0
data_flag = False
resource_package = 'investpy'
resource_path = '/'.join(('resources', 'bonds.csv'))
if pkg_resources.resource_exists(resource_package, resource_path):
bonds = pd.read_csv(
pkg_resources.resource_filename(resource_package, resource_path))
else:
raise FileNotFoundError("ERR#0064: bonds file not found or errored.")
if bonds is None:
raise IOError(
"ERR#0065: bonds object not found or unable to retrieve.")
bond = unidecode(bond.strip().lower())
if bond not in list(bonds['name'].str.lower()):
raise RuntimeError("ERR#0068: bond " + bond +
" not found, check if it is correct.")
id_ = bonds.loc[(bonds['name'].str.lower() == bond).idxmax(), 'id']
name = bonds.loc[(bonds['name'].str.lower() == bond).idxmax(), 'name']
full_name = bonds.loc[(bonds['name'].str.lower() == bond).idxmax(),
'full_name']
final = list()
header = full_name + " Bond Yield Historical Data"
for index in range(len(date_interval['intervals'])):
interval_counter += 1
params = {
"curr_id": id_,
"smlID": str(randint(1000000, 99999999)),
"header": header,
"st_date": date_interval['intervals'][index]['start'],
"end_date": date_interval['intervals'][index]['end'],
"interval_sec": interval.capitalize(),
"sort_col": "date",
"sort_ord": "DESC",
"action": "historical_data"
}
head = {
"User-Agent": random_user_agent(),
"X-Requested-With": "XMLHttpRequest",
"Accept": "text/html",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
}
url = "https://www.investing.com/instruments/HistoricalDataAjax"
req = requests.post(url, headers=head, data=params)
if req.status_code != 200:
raise ConnectionError("ERR#0015: error " + str(req.status_code) +
", try again later.")
if not req.text:
continue
root_ = fromstring(req.text)
path_ = root_.xpath(".//table[@id='curr_table']/tbody/tr")
result = list()
if path_:
for elements_ in path_:
if elements_.xpath(
".//td")[0].text_content() == 'No results found':
if interval_counter < interval_limit:
data_flag = False
else:
raise IndexError(
"ERR#0069: bond information unavailable or not found."
)
else:
data_flag = True
if data_flag is True:
info = []
for nested_ in elements_.xpath(".//td"):
info.append(nested_.get('data-real-value'))
bond_date = datetime.strptime(
str(
datetime.fromtimestamp(
int(info[0]), tz=pytz.timezone('GMT')).date()),
'%Y-%m-%d')
bond_close = float(info[1].replace(',', ''))
bond_open = float(info[2].replace(',', ''))
bond_high = float(info[3].replace(',', ''))
bond_low = float(info[4].replace(',', ''))
result.insert(
len(result),
Data(bond_date, bond_open, bond_high, bond_low,
bond_close, None, None, None))
if data_flag is True:
if order in ['ascending', 'asc']:
result = result[::-1]
elif order in ['descending', 'desc']:
result = result
if as_json is True:
json_list = [value.bond_as_json() for value in result]
final.append(json_list)
elif as_json is False:
df = pd.DataFrame.from_records(
[value.bond_to_dict() for value in result])
df.set_index('Date', inplace=True)
final.append(df)
else:
raise RuntimeError(
"ERR#0004: data retrieval error while scraping.")
if order in ['descending', 'desc']:
final.reverse()
if as_json is True:
json_ = {
'name': name,
'historical':
[value for json_list in final for value in json_list]
}
return json.dumps(json_, sort_keys=False)
elif as_json is False:
return pd.concat(final)
def __init__(self,
sc=None,
app_name="Hail",
master=None,
local='local[*]',
log=None,
quiet=False,
append=False,
min_block_size=1,
branching_factor=50,
tmp_dir=None,
default_reference="GRCh37",
idempotent=False,
global_seed=6348563392232659379,
_backend=None):
if Env._hc:
if idempotent:
return
else:
raise FatalError(
'Hail has already been initialized, restart session '
'or stop Hail to change configuration.')
if pkg_resources.resource_exists(__name__, "hail-all-spark.jar"):
hail_jar_path = pkg_resources.resource_filename(
__name__, "hail-all-spark.jar")
assert os.path.exists(
hail_jar_path), f'{hail_jar_path} does not exist'
sys.stderr.write(f'using hail jar at {hail_jar_path}\n')
conf = SparkConf()
conf.set('spark.driver.extraClassPath', hail_jar_path)
conf.set('spark.executor.extraClassPath', hail_jar_path)
SparkContext._ensure_initialized(conf=conf)
else:
SparkContext._ensure_initialized()
self._gateway = SparkContext._gateway
self._jvm = SparkContext._jvm
# hail package
self._hail = getattr(self._jvm, 'is').hail
self._warn_cols_order = True
self._warn_entries_order = True
Env._jvm = self._jvm
Env._gateway = self._gateway
jsc = sc._jsc.sc() if sc else None
if _backend is None:
apiserver_url = os.environ.get('HAIL_APISERVER_URL')
if apiserver_url is not None:
_backend = ServiceBackend(apiserver_url)
else:
_backend = SparkBackend()
self._backend = _backend
tmp_dir = get_env_or_default(tmp_dir, 'TMPDIR', '/tmp')
version = read_version_info()
hail.__version__ = version
if log is None:
log = hail.utils.timestamp_path(os.path.join(os.getcwd(), 'hail'),
suffix=f'-{version}.log')
self._log = log
# we always pass 'quiet' to the JVM because stderr output needs
# to be routed through Python separately.
# if idempotent:
if idempotent:
self._jhc = self._hail.HailContext.getOrCreate(
jsc, app_name, joption(master), local, log, True, append,
min_block_size, branching_factor, tmp_dir)
else:
self._jhc = self._hail.HailContext.apply(jsc, app_name,
joption(master), local,
log, True, append,
min_block_size,
branching_factor, tmp_dir)
self._jsc = self._jhc.sc()
self.sc = sc if sc else SparkContext(
gateway=self._gateway, jsc=self._jvm.JavaSparkContext(self._jsc))
self._jsql_context = self._jhc.sqlContext()
self._sql_context = SQLContext(self.sc, jsqlContext=self._jsql_context)
super(HailContext, self).__init__()
# do this at the end in case something errors, so we don't raise the above error without a real HC
Env._hc = self
ReferenceGenome._from_config(_backend.get_reference('GRCh37'), True)
ReferenceGenome._from_config(_backend.get_reference('GRCh38'), True)
ReferenceGenome._from_config(_backend.get_reference('GRCm38'), True)
if default_reference in ReferenceGenome._references:
self._default_ref = ReferenceGenome._references[default_reference]
else:
self._default_ref = ReferenceGenome.read(default_reference)
jar_version = self._jhc.version()
if jar_version != version:
raise RuntimeError(
f"Hail version mismatch between JAR and Python library\n"
f" JAR: {jar_version}\n"
f" Python: {version}")
if not quiet:
sys.stderr.write('Running on Apache Spark version {}\n'.format(
self.sc.version))
if self._jsc.uiWebUrl().isDefined():
sys.stderr.write('SparkUI available at {}\n'.format(
self._jsc.uiWebUrl().get()))
connect_logger('localhost', 12888)
self._hail.HailContext.startProgressBar(self._jsc)
sys.stderr.write(
'Welcome to\n'
' __ __ <>__\n'
' / /_/ /__ __/ /\n'
' / __ / _ `/ / /\n'
' /_/ /_/\\_,_/_/_/ version {}\n'.format(version))
if version.startswith('devel'):
sys.stderr.write(
'NOTE: This is a beta version. Interfaces may change\n'
' during the beta period. We recommend pulling\n'
' the latest changes weekly.\n')
sys.stderr.write(f'LOGGING: writing to {log}\n')
install_exception_handler()
Env.set_seed(global_seed)
def has_gff3(locus, segment, collection=None):
file_name = _get_file_name(locus,
segment,
collection=collection,
extension='.gff3')
return resource_exists(_PKG, file_name)
def main(args, pass_through_args):
conf = ClusterConfig()
conf.extend_flag('image-version', IMAGE_VERSION)
if not pkg_resources.resource_exists('hailtop.hailctl', "deploy.yaml"):
raise RuntimeError("package has no 'deploy.yaml' file")
deploy_metadata = yaml.safe_load(
pkg_resources.resource_stream('hailtop.hailctl',
"deploy.yaml"))['dataproc']
conf.extend_flag('properties', DEFAULT_PROPERTIES)
if args.properties:
conf.parse_and_extend('properties', args.properties)
if args.debug_mode:
conf.extend_flag(
'properties', {
"spark:spark.driver.extraJavaOptions":
"-Xss4M -XX:+HeapDumpOnOutOfMemoryError",
"spark:spark.executor.extraJavaOptions":
"-Xss4M -XX:+HeapDumpOnOutOfMemoryError",
})
# default to highmem machines if using VEP
if not args.worker_machine_type:
args.worker_machine_type = 'n1-highmem-8' if args.vep else 'n1-standard-8'
# default initialization script to start up cluster with
conf.extend_flag('initialization-actions',
[deploy_metadata['init_notebook.py']])
# requester pays support
if args.requester_pays_allow_all or args.requester_pays_allow_buckets or args.requester_pays_allow_annotation_db:
if args.requester_pays_allow_all and args.requester_pays_allow_buckets:
raise RuntimeError(
"Cannot specify both 'requester_pays_allow_all' and 'requester_pays_allow_buckets"
)
if args.requester_pays_allow_all:
requester_pays_mode = "AUTO"
else:
requester_pays_mode = "CUSTOM"
requester_pays_bucket_sources = []
if args.requester_pays_allow_buckets:
requester_pays_bucket_sources.append(
args.requester_pays_allow_buckets)
if args.requester_pays_allow_annotation_db:
requester_pays_bucket_sources.extend(ANNOTATION_DB_BUCKETS)
conf.extend_flag(
"properties", {
"spark:spark.hadoop.fs.gs.requester.pays.buckets":
",".join(requester_pays_bucket_sources)
})
# Need to pick requester pays project.
requester_pays_project = args.project if args.project else gcloud.get_config(
"project")
conf.extend_flag(
"properties", {
"spark:spark.hadoop.fs.gs.requester.pays.mode":
requester_pays_mode,
"spark:spark.hadoop.fs.gs.requester.pays.project.id":
requester_pays_project
})
# gcloud version 277 and onwards requires you to specify a region. Let's just require it for all hailctl users for consistency.
if args.region:
project_region = args.region
else:
project_region = gcloud.get_config("dataproc/region")
if not project_region:
raise RuntimeError(
"Could not determine dataproc region. Use --region argument to hailctl, or use `gcloud config set dataproc/region <my-region>` to set a default."
)
# add VEP init script
if args.vep:
# VEP is too expensive if you have to pay egress charges. We must choose the right replicate.
replicate = REGION_TO_REPLICATE_MAPPING.get(project_region)
if replicate is None:
raise RuntimeError(
f"The --vep argument is not currently provided in your region.\n"
f" Please contact the Hail team on https://discuss.hail.is for support.\n"
f" Your region: {project_region}\n"
f" Supported regions: {', '.join(REGION_TO_REPLICATE_MAPPING.keys())}"
)
print(f"Pulling VEP data from bucket in {replicate}.")
conf.extend_flag('metadata', {"VEP_REPLICATE": replicate})
vep_config_path = "/vep_data/vep-gcloud.json"
conf.extend_flag(
'metadata', {
"VEP_CONFIG_PATH": vep_config_path,
"VEP_CONFIG_URI": f"file://{vep_config_path}"
})
conf.extend_flag('initialization-actions',
[deploy_metadata[f'vep-{args.vep}.sh']])
# add custom init scripts
if args.init:
conf.extend_flag('initialization-actions', args.init.split(','))
if args.metadata:
conf.parse_and_extend('metadata', args.metadata)
wheel = args.wheel or deploy_metadata['wheel']
conf.extend_flag('metadata', {'WHEEL': wheel})
# if Python packages requested, add metadata variable
packages = deploy_metadata['pip_dependencies'].strip('|').split('|||')
metadata_pkgs = conf.flags['metadata'].get('PKGS')
split_regex = r'[|,]'
if metadata_pkgs:
packages.extend(re.split(split_regex, metadata_pkgs))
if args.packages:
packages.extend(re.split(split_regex, args.packages))
conf.extend_flag('metadata', {'PKGS': '|'.join(set(packages))})
def disk_size(size):
if args.vep:
size = max(size, 200)
return str(size) + 'GB'
conf.extend_flag(
'properties', {
"spark:spark.driver.memory":
"{driver_memory}g".format(driver_memory=str(
int(MACHINE_MEM[args.master_machine_type] *
args.master_memory_fraction)))
})
conf.flags['master-machine-type'] = args.master_machine_type
conf.flags['master-boot-disk-size'] = '{}GB'.format(
args.master_boot_disk_size)
conf.flags['num-master-local-ssds'] = args.num_master_local_ssds
conf.flags['num-secondary-workers'] = args.num_secondary_workers
conf.flags['num-worker-local-ssds'] = args.num_worker_local_ssds
conf.flags['num-workers'] = args.num_workers
conf.flags['secondary-worker-boot-disk-size'] = disk_size(
args.secondary_worker_boot_disk_size)
conf.flags['worker-boot-disk-size'] = disk_size(args.worker_boot_disk_size)
conf.flags['worker-machine-type'] = args.worker_machine_type
if args.region:
conf.flags['region'] = args.region
if args.zone:
conf.flags['zone'] = args.zone
conf.flags['initialization-action-timeout'] = args.init_timeout
if args.network:
conf.flags['network'] = args.network
if args.configuration:
conf.flags['configuration'] = args.configuration
if args.project:
conf.flags['project'] = args.project
if args.bucket:
conf.flags['bucket'] = args.bucket
account = gcloud.get_config("account")
if account:
conf.flags['labels'] = 'creator=' + re.sub(r'[^0-9a-z_\-]', '_',
account.lower())[:63]
# rewrite metadata and properties to escape them
conf.flags['metadata'] = '^|||^' + '|||'.join(
f'{k}={v}' for k, v in conf.flags['metadata'].items())
conf.flags['properties'] = '^|||^' + '|||'.join(
f'{k}={v}' for k, v in conf.flags['properties'].items())
# command to start cluster
cmd = conf.get_command(args.name)
if args.beta or args.max_idle or args.max_age:
cmd.insert(1, 'beta')
if args.max_idle:
cmd.append('--max-idle={}'.format(args.max_idle))
if args.max_age:
cmd.append('--max-age={}'.format(args.max_age))
if args.expiration_time:
cmd.append('--expiration_time={}'.format(args.expiration_time))
cmd.extend(pass_through_args)
# print underlying gcloud command
print(' '.join(cmd[:5]) + ' \\\n ' + ' \\\n '.join(cmd[5:]))
# spin up cluster
if not args.dry_run:
print("Starting cluster '{}'...".format(args.name))
gcloud.run(cmd[1:])
if args.master_tags:
add_tags_command = [
'compute', 'instances', 'add-tags', args.name + '-m', '--tags',
args.master_tags
]
if args.project:
add_tags_command.append(f"--project={args.project}")
if args.zone:
add_tags_command.append(f"--zone={args.zone}")
print('gcloud ' + ' '.join(add_tags_command))
if not args.dry_run:
gcloud.run(add_tags_command)
def get_bond_recent_data(bond, as_json=False, order='ascending', interval='Daily'):
"""
This function retrieves recent historical data from the introduced bond from Investing.com. So on, the recent data
of the introduced bond will be retrieved and returned as a :obj:`pandas.DataFrame` if the parameters are valid and
the request to Investing.com succeeds. Note that additionally some optional parameters can be specified: as_json
and order, which let the user decide if the data is going to be returned as a :obj:`json` or not, and if the recent
data is going to be ordered ascending or descending (where the index is the date), respectively.
Args:
bond (:obj:`str`): name of the bond to retrieve recent historical data from.
as_json (:obj:`bool`, optional):
to determine the format of the output data, either a :obj:`pandas.DataFrame` if False and a :obj:`json` if True.
order (:obj:`str`, optional): to define the order of the retrieved data which can either be ascending or descending.
interval (:obj:`str`, optional):
value to define the historical data interval to retrieve, by default `Daily`, but it can also be `Weekly` or `Monthly`.
Returns:
:obj:`pandas.DataFrame` or :obj:`json`:
The function can return either a :obj:`pandas.DataFrame` or a :obj:`json` object, containing the retrieved
recent data of the specified bond. So on, the resulting dataframe contains the open, high, low and close
values for the selected bond on market days.
The resulting recent data, in case that the default parameters were applied, will look like::
Date || Open | High | Low | Close
-----||------|------|-----|-------
xxxx || xxxx | xxxx | xxx | xxxxx
but in case that as_json parameter was defined as True, then the output will be::
{
name: name,
recent: [
{
date: 'dd/mm/yyyy',
open: x,
high: x,
low: x,
close: x
},
...
]
}
Raises:
ValueError: raised whenever any of the introduced arguments is not valid or errored.
IOError: raised if bonds object/file was not found or unable to retrieve.
RuntimeError: raised if the introduced bond was not found or did not match any of the existing ones.
ConnectionError: raised if connection to Investing.com could not be established.
IndexError: raised if bond historical data was unavailable or not found in Investing.com.
Examples:
>>> investpy.get_bond_recent_data(bond='Argentina 3Y')
Open High Low Close
Date
2019-09-23 52.214 52.214 52.214 52.214
2019-09-24 52.323 52.323 52.323 52.323
2019-09-25 52.432 52.432 52.432 52.432
2019-09-26 52.765 52.765 52.765 52.765
2019-09-27 52.876 52.876 52.876 52.876
"""
if not bond:
raise ValueError("ERR#0066: bond parameter is mandatory and must be a valid bond name.")
if not isinstance(bond, str):
raise ValueError("ERR#0067: bond argument needs to be a str.")
if not isinstance(as_json, bool):
raise ValueError("ERR#0002: as_json argument can just be True or False, bool type.")
if order not in ['ascending', 'asc', 'descending', 'desc']:
raise ValueError("ERR#0003: order argument can just be ascending (asc) or descending (desc), str type.")
if not interval:
raise ValueError("ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'.")
if not isinstance(interval, str):
raise ValueError("ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'.")
if interval not in ['Daily', 'Weekly', 'Monthly']:
raise ValueError("ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'.")
resource_package = 'investpy'
resource_path = '/'.join(('resources', 'bonds', 'bonds.csv'))
if pkg_resources.resource_exists(resource_package, resource_path):
bonds = pd.read_csv(pkg_resources.resource_filename(resource_package, resource_path))
else:
raise FileNotFoundError("ERR#0064: bonds file not found or errored.")
if bonds is None:
raise IOError("ERR#0065: bonds object not found or unable to retrieve.")
bond = bond.strip()
bond = bond.lower()
if unidecode.unidecode(bond) not in [unidecode.unidecode(value.lower()) for value in bonds['name'].tolist()]:
raise RuntimeError("ERR#0068: bond " + bond + " not found, check if it is correct.")
id_ = bonds.loc[(bonds['name'].str.lower() == bond).idxmax(), 'id']
name = bonds.loc[(bonds['name'].str.lower() == bond).idxmax(), 'name']
full_name = bonds.loc[(bonds['name'].str.lower() == bond).idxmax(), 'full_name']
header = full_name + " Bond Yield Historical Data"
params = {
"curr_id": id_,
"smlID": str(randint(1000000, 99999999)),
"header": header,
"interval_sec": interval,
"sort_col": "date",
"sort_ord": "DESC",
"action": "historical_data"
}
head = {
"User-Agent": get_random(),
"X-Requested-With": "XMLHttpRequest",
"Accept": "text/html",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
}
url = "https://www.investing.com/instruments/HistoricalDataAjax"
req = requests.post(url, headers=head, data=params)
if req.status_code != 200:
raise ConnectionError("ERR#0015: error " + str(req.status_code) + ", try again later.")
root_ = fromstring(req.text)
path_ = root_.xpath(".//table[@id='curr_table']/tbody/tr")
result = list()
if path_:
for elements_ in path_:
if elements_.xpath(".//td")[0].text_content() == 'No results found':
raise IndexError("ERR#0069: bond information unavailable or not found.")
info = []
for nested_ in elements_.xpath(".//td"):
info.append(nested_.get('data-real-value'))
bond_date = datetime.fromtimestamp(int(info[0]))
bond_date = date(bond_date.year, bond_date.month, bond_date.day)
bond_close = float(info[1].replace(',', ''))
bond_open = float(info[2].replace(',', ''))
bond_high = float(info[3].replace(',', ''))
bond_low = float(info[4].replace(',', ''))
result.insert(len(result),
Data(bond_date, bond_open, bond_high, bond_low, bond_close, None, None, None))
if order in ['ascending', 'asc']:
result = result[::-1]
elif order in ['descending', 'desc']:
result = result
if as_json is True:
json_ = {
'name': name,
'recent':
[value.bond_as_json() for value in result]
}
return json.dumps(json_, sort_keys=False)
elif as_json is False:
df = pd.DataFrame.from_records([value.bond_to_dict() for value in result])
df.set_index('Date', inplace=True)
return df
else:
raise RuntimeError("ERR#0004: data retrieval error while scraping.")
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import sys
import pkg_resources as pr
import os
import subprocess
# Initialize binaries
if sys.platform[0:5] == 'linux':
assert (pr.resource_exists('lomap', 'binaries/linux/ltl2ba'))
assert (pr.resource_exists('lomap', 'binaries/linux/scheck2'))
ltl2ba_binary = pr.resource_filename('lomap', 'binaries/linux/ltl2ba')
scheck_binary = pr.resource_filename('lomap', 'binaries/linux/scheck2')
elif sys.platform == 'darwin':
assert (pr.resource_exists('lomap', 'binaries/mac/ltl2ba'))
assert (pr.resource_exists('lomap', 'binaries/mac/scheck2'))
ltl2ba_binary = pr.resource_filename('lomap', 'binaries/mac/ltl2ba')
scheck_binary = pr.resource_filename('lomap', 'binaries/mac/scheck2')
else:
print >> sys.stderr, sys.platform, 'platform not supported yet!'
print >> sys.stderr, 'Binaries will not work!'
ltl2ba_binary = None
scheck_binary = None
# Check and fix chmod as needed
def currency_crosses_as_dict(base=None,
second=None,
columns=None,
as_json=False):
"""
This function retrieves all the available currency crosses from Investing.com and returns them as a
:obj:`dict`, which contains not just the currency crosses names, but all the fields contained on
the currency_crosses file is columns is None, otherwise, just the specified column values will be returned. Note
that the filtering params are both base and second, which mean the base and the second currency of the currency
cross, for example, in the currency cross `EUR/USD` the base currency is EUR and the second currency is USD. These
are optional parameters, so specifying one of them means that all the currency crosses where the introduced
currency is either base or second will be returned; if both are specified, just the introduced currency cross will
be returned if it exists. All the available currency crosses can be found at: https://www.investing.com/currencies/
Args:
base (:obj:`str`, optional):
symbol of the base currency of the currency cross, this will return a :obj:`pandas.DataFrame` containing
all the currency crosses where the base currency matches the introduced one.
second (:obj:`str`):
symbol of the second currency of the currency cross, this will return a :obj:`pandas.DataFrame` containing
all the currency crosses where the second currency matches the introduced one.
columns (:obj:`list`, optional):
names of the columns of the currency crosses data to retrieve <name, full_name, base, base_name,
second, second_name>
as_json (:obj:`bool`, optional):
value to determine the format of the output data which can either be a :obj:`dict` or a :obj:`json`.
Returns:
:obj:`dict` or :obj:`json` - currency_crosses_dict:
The resulting :obj:`dict` contains the retrieved data if found, if not, the corresponding
fields are filled with `None` values.
In case the information was successfully retrieved, the :obj:`dict` will look like::
currency_crosses_dict = {
'name': name,
'full_name': full_name,
'base': base,
'base_name': base_name,
'second': second,
'second_name': second_name
}
Raises:
ValueError: raised if any of the introduced arguments is not valid or errored.
FileNotFoundError: raised if `currency_crosses.csv` file was not found.
IOError: raised if currency crosses retrieval failed, both for missing file or empty file.
"""
if base is not None and not isinstance(base, str):
raise ValueError(
"ERR#0049: specified base currency value is not valid.")
if second is not None and not isinstance(second, str):
raise ValueError(
"ERR#0051: specified second currency value is not valid.")
if not isinstance(as_json, bool):
raise ValueError(
"ERR#0002: as_json argument can just be True or False, bool type.")
resource_package = 'investpy'
resource_path = '/'.join(('resources', 'currency_crosses.csv'))
if pkg_resources.resource_exists(resource_package, resource_path):
currency_crosses = pd.read_csv(pkg_resources.resource_filename(
resource_package, resource_path),
keep_default_na=False)
else:
raise FileNotFoundError(
"ERR#0060: currency_crosses file not found or errored.")
if currency_crosses is None:
raise IOError(
"ERR#0050: currency_crosses not found or unable to retrieve.")
available_currencies = available_currencies_as_list()
currency_crosses.drop(columns=['tag', 'id'], inplace=True)
currency_crosses = currency_crosses.where(pd.notnull(currency_crosses),
None)
if columns is None:
columns = currency_crosses.columns.tolist()
else:
if not isinstance(columns, list):
raise ValueError(
"ERR#0020: specified columns argument is not a list, it can just be list type."
)
if not all(column in currency_crosses.columns.tolist()
for column in columns):
raise ValueError(
"ERR#0021: specified columns does not exist, available columns are "
"<name, full_name, base, base_name, second, second_name>")
if base is None and second is None:
currency_crosses.reset_index(drop=True, inplace=True)
if as_json:
return json.dumps(
currency_crosses[columns].to_dict(orient='records'))
else:
return currency_crosses[columns].to_dict(orient='records')
elif base is not None:
base = unidecode(base.strip().upper())
if base not in available_currencies:
raise ValueError("ERR#0053: the introduced currency " + base +
" does not exists.")
if second is not None:
second = unidecode(second.strip().upper())
if second not in available_currencies:
raise ValueError("ERR#0053: the introduced currency " +
second + " does not exists.")
currency_crosses = currency_crosses[
(currency_crosses['base'] == base)
& (currency_crosses['second'] == second)]
currency_crosses.reset_index(drop=True, inplace=True)
if len(currency_crosses) > 0:
if as_json:
return json.dumps(
currency_crosses[columns].to_dict(orient='records'))
else:
return currency_crosses[columns].to_dict(orient='records')
else:
raise ValueError("ERR#0054: the introduced currency cross " +
base + "/" + second + " does not exists.")
else:
currency_crosses = currency_crosses[currency_crosses['base'] ==
base]
currency_crosses.reset_index(drop=True, inplace=True)
if as_json:
return json.dumps(
currency_crosses[columns].to_dict(orient='records'))
else:
return currency_crosses[columns].to_dict(orient='records')
elif second is not None:
second = unidecode(second.strip().upper())
if second not in available_currencies:
raise ValueError("ERR#0053: the introduced currency " + second +
" does not exists.")
currency_crosses = currency_crosses[currency_crosses['second'] ==
unidecode(second.upper())]
currency_crosses.reset_index(drop=True, inplace=True)
if as_json:
return json.dumps(
currency_crosses[columns].to_dict(orient='records'))
else:
return currency_crosses[columns].to_dict(orient='records')
def _has_resource(self, resource):
return pkg_resources.resource_exists(__name__, resource.name)
def __init__(self, params, status, calib_data, service_name):
"""
:param params: pywws configuration.
:type params: :class:`pywws.DataStore.params`
:param status: pywws status store.
:type status: :class:`pywws.DataStore.status`
:param calib_data: 'calibrated' data.
:type calib_data: :class:`pywws.DataStore.calib_store`
:param service_name: name of service to upload to.
:type service_name: string
"""
self.logger = logging.getLogger('pywws.ToService(%s)' % service_name)
self.params = params
self.status = status
self.data = calib_data
self.service_name = service_name
# 'derived' services such as 'underground_rf' share their
# parent's config and templates
config_section = self.service_name.split('_')[0]
if config_section == self.service_name:
self.parent = None
else:
self.parent = config_section
self.old_response = None
self.old_ex = None
# set default socket timeout, so urlopen calls don't hang forever
socket.setdefaulttimeout(30)
# open params file
service_params = SafeConfigParser()
service_params.optionxform = str
service_params.readfp(
pkg_resources.resource_stream(
'pywws', 'services/%s.ini' % (self.service_name)))
# get URL
self.server = service_params.get('config', 'url')
parsed_url = urlparse.urlsplit(self.server)
if parsed_url.scheme == 'aprs':
self.send_data = self.aprs_send_data
server, port = parsed_url.netloc.split(':')
self.server = (server, int(port))
elif parsed_url.scheme == 'mqtt':
self.send_data = self.mqtt_send_data
else:
self.send_data = self.http_send_data
self.use_get = eval(service_params.get('config', 'use get'))
# get fixed part of upload data
self.fixed_data = dict()
for name, value in service_params.items('fixed'):
if value[0] == '*':
value = self.params.get(config_section, value[1:], 'unknown')
self.fixed_data[name] = value
# create templater
self.templater = Template.Template(self.params,
self.status,
self.data,
self.data,
None,
None,
use_locale=False)
template_name = 'services/%s_template_%s.txt' % (
config_section, self.params.get('config', 'ws type'))
if not pkg_resources.resource_exists('pywws', template_name):
template_name = 'services/%s_template_1080.txt' % (config_section)
self.template_file = pkg_resources.resource_stream(
'pywws', template_name)
# get other parameters
self.auth_type = service_params.get('config', 'auth_type')
if self.auth_type == 'basic':
user = self.params.get(config_section, 'user', 'unknown')
password = self.params.get(config_section, 'password', 'unknown')
self.auth = 'Basic %s' % base64.b64encode('%s:%s' %
(user, password))
self.catchup = eval(service_params.get('config', 'catchup'))
self.expected_result = eval(service_params.get('config', 'result'))
self.interval = eval(service_params.get('config', 'interval'))
self.interval = max(self.interval, 40)
self.interval = timedelta(seconds=self.interval)
# move 'last update' from params to status
last_update = self.params.get_datetime(self.service_name,
'last update')
if last_update:
self.params.unset(self.service_name, 'last update')
self.status.set('last update', self.service_name,
last_update.isoformat(' '))
# set timestamp of first data to upload
self.next_update = datetime.utcnow() - max(
timedelta(days=self.catchup), self.interval)
def currency_crosses_as_df(base=None, second=None):
"""
This function retrieves all the available currency crosses from Investing.com and returns them as a
:obj:`pandas.DataFrame`, which contains not just the currency crosses names, but all the fields contained on
the currency_crosses file. Note that the filtering params are both base and second, which mean the base and the
second currency of the currency cross, for example, in the currency cross `EUR/USD` the base currency is EUR and
the second currency is USD. These are optional parameters, so specifying one of them means that all the currency
crosses where the introduced currency is either base or second will be returned; if both are specified,
just the introduced currency cross will be returned if it exists. All the available currency crosses can be found
at: https://www.investing.com/currencies/
Args:
base (:obj:`str`, optional):
symbol of the base currency of the currency cross, this will return a :obj:`pandas.DataFrame` containing
all the currency crosses where the base currency matches the introduced one.
second (:obj:`str`):
symbol of the second currency of the currency cross, this will return a :obj:`pandas.DataFrame` containing
all the currency crosses where the second currency matches the introduced one.
Returns:
:obj:`pandas.DataFrame` - currency_crosses_df:
The resulting :obj:`pandas.DataFrame` contains all the currency crosses basic information retrieved from
Investing.com.
In case the information was successfully retrieved, the resulting :obj:`pandas.DataFrame` will look like::
name | full_name | base | second | base_name | second_name
-----|-----------|------|--------|-----------|-------------
xxxx | xxxxxxxxx | xxxx | xxxxxx | xxxxxxxxx | xxxxxxxxxxx
Raises:
ValueError: raised if any of the introduced arguments is not valid or errored.
FileNotFoundError: raised if `currency_crosses.csv` file was not found.
IOError: raised if currency crosses retrieval failed, both for missing file or empty file.
"""
if base is not None and not isinstance(base, str):
raise ValueError(
"ERR#0049: specified base currency value is not valid.")
if second is not None and not isinstance(second, str):
raise ValueError(
"ERR#0051: specified second currency value is not valid.")
resource_package = 'investpy'
resource_path = '/'.join(('resources', 'currency_crosses.csv'))
if pkg_resources.resource_exists(resource_package, resource_path):
currency_crosses = pd.read_csv(pkg_resources.resource_filename(
resource_package, resource_path),
keep_default_na=False)
else:
raise FileNotFoundError(
"ERR#0060: currency_crosses file not found or errored.")
if currency_crosses is None:
raise IOError(
"ERR#0050: currency_crosses not found or unable to retrieve.")
available_currencies = available_currencies_as_list()
currency_crosses.drop(columns=['tag', 'id'], inplace=True)
currency_crosses = currency_crosses.where(pd.notnull(currency_crosses),
None)
if base is None and second is None:
currency_crosses.reset_index(drop=True, inplace=True)
return currency_crosses
elif base is not None:
base = unidecode(base.strip().upper())
if base not in available_currencies:
raise ValueError("ERR#0053: the introduced currency " + base +
" does not exists.")
if second is not None:
second = unidecode(second.strip().upper())
if second not in available_currencies:
raise ValueError("ERR#0053: the introduced currency " +
second + " does not exists.")
currency_crosses = currency_crosses[
(currency_crosses['base'] == base)
& (currency_crosses['second'] == second)]
currency_crosses.reset_index(drop=True, inplace=True)
if len(currency_crosses) > 0:
return currency_crosses
else:
raise ValueError("ERR#0054: the introduced currency cross " +
base + "/" + second + " does not exists.")
else:
currency_crosses = currency_crosses[currency_crosses['base'] ==
base]
currency_crosses.reset_index(drop=True, inplace=True)
return currency_crosses
elif second is not None:
second = unidecode(second.strip().upper())
if second not in available_currencies:
raise ValueError("ERR#0053: the introduced currency " + second +
" does not exists.")
currency_crosses = currency_crosses[currency_crosses['second'] ==
second]
currency_crosses.reset_index(drop=True, inplace=True)
return currency_crosses
from logging import getLogger
from ConfigParser import ConfigParser
from locale import getdefaultlocale
from pkg_resources import resource_filename, resource_exists
from codecs import open
try:
from ..comm import notif
except ValueError:
from comm import notif
try:
from ..arping import arping, arpingerror
except ValueError:
from arping import arping, arpingerror
# Set language messages to use
if resource_exists("homenetcontrol",
"langs/" + getdefaultlocale()[0][:2] + ".lang"):
langfile = resource_filename(
"homenetcontrol", "langs/" + getdefaultlocale()[0][:2] + ".lang")
else:
langfile = resource_filename("homenetcontrol", "langs/en.lang")
lang = ConfigParser()
lang.readfp(open(langfile, "r", "utf8"))
# Get info from config file
if resource_exists("homenetcontrol", "check_devices/config.ini"):
configfile = resource_filename("homenetcontrol",
"check_devices/config.ini")
else:
configfile = "/etc/homenetcontrol/config.ini"
config = ConfigParser()
config.optionxform = str
def get_commodity_information(commodity, country=None, as_json=False):
"""
This function retrieves fundamental financial information from the specified commodity. The retrieved
information from the commodity can be valuable as it is additional information that can be used combined
with OHLC values, so to determine financial insights from the company which holds the specified commodity.
Args:
commodity (:obj:`str`): name of the commodity to retrieve information from.
country (:obj:`str`, optional):
name of the country to retrieve the commodity information from (if there is more than one country
that provides data from the same commodity).
as_json (:obj:`bool`, optional):
optional argument to determine the format of the output data (:obj:`dict` or :obj:`json`).
Returns:
:obj:`pandas.DataFrame` or :obj:`dict`- commodity_information:
The resulting :obj:`pandas.DataFrame` contains the information fields retrieved from Investing.com
from the specified commodity; it can also be returned as a :obj:`dict`, if argument `as_json=True`.
If any of the information fields could not be retrieved, that field/s will be filled with
None values. If the retrieval process succeeded, the resulting :obj:`dict` will look like::
commodity_information = {
"1-Year Change": "16.15%",
"52 wk Range": "1,270.2 - 1,566.2",
"Base Symbol": "GC",
"Commodity Name": "Gold",
"Contract Size": "100 Troy Ounces",
"Last Rollover Day": "24/11/2019",
"Month": "Feb 20",
"Months": "GJMQVZ",
"Open": 1479.8,
"Point Value": "$100",
"Prev. Close": 1481.2,
"Settlement Day": "25/01/2020",
"Settlement Type": "Physical",
"Tick Size": 0.1,
"Tick Value": 10.0,
"Todays Range": "1,477.55 - 1,484.25"
}
Raises:
ValueError: raised if any of the introduced arguments is not valid or errored.
FileNotFoundError: raised if `commodities.csv` file was not found or errored.
IOError: raised if commodities.csv file is empty or errored.
RuntimeError: raised if scraping process failed while running.
ConnectionError: raised if the connection to Investing.com errored (did not return HTTP 200)
"""
if not commodity:
raise ValueError(
"ERR#0078: commodity parameter is mandatory and must be a valid commodity name."
)
if not isinstance(commodity, str):
raise ValueError(
"ERR#0078: commodity parameter is mandatory and must be a valid commodity name."
)
if country is not None and not isinstance(country, str):
raise ValueError("ERR#0025: specified country value not valid.")
if not isinstance(as_json, bool):
raise ValueError(
"ERR#0002: as_json argument can just be True or False, bool type.")
resource_package = 'investpy'
resource_path = '/'.join(('resources', 'commodities.csv'))
if pkg_resources.resource_exists(resource_package, resource_path):
commodities = pd.read_csv(
pkg_resources.resource_filename(resource_package, resource_path))
else:
raise FileNotFoundError(
"ERR#0075: commodities file not found or errored.")
if commodities is None:
raise IOError("ERR#0076: commodities not found or unable to retrieve.")
commodity = commodity.strip()
commodity = commodity.lower()
if unidecode(commodity) not in [
unidecode(value.lower()) for value in commodities['name'].tolist()
]:
raise RuntimeError("ERR#0079: commodity " + commodity +
" not found, check if it is correct.")
if country is None:
found_commodities = commodities[commodities['name'].str.lower() ==
commodity]
if len(found_commodities) > 1:
msg = "Note that the displayed commodity information can differ depending on the country. " \
"If you want to retrieve " + commodity + " data from either " + \
" or ".join(found_commodities['country'].tolist()) + ", specify the country parameter."
warnings.warn(msg, Warning)
del found_commodities
else:
if unidecode(country.lower()) not in commodities['country'].unique(
).tolist():
raise RuntimeError("ERR#0034: country " + country.lower() +
" not found, check if it is correct.")
commodities = commodities[commodities['country'] == unidecode(
country.lower())]
name = commodities.loc[(
commodities['name'].str.lower() == commodity).idxmax(), 'name']
tag = commodities.loc[(
commodities['name'].str.lower() == commodity).idxmax(), 'tag']
url = "https://www.investing.com/commodities/" + tag
head = {
"User-Agent": random_user_agent(),
"X-Requested-With": "XMLHttpRequest",
"Accept": "text/html",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
}
req = requests.get(url, headers=head)
if req.status_code != 200:
raise ConnectionError("ERR#0015: error " + str(req.status_code) +
", try again later.")
root_ = fromstring(req.text)
path_ = root_.xpath("//div[contains(@class, 'overviewDataTable')]/div")
result = pd.DataFrame(columns=[
'Commodity Name', 'Prev. Close', 'Month', 'Tick Size', 'Open',
'Contract Size', 'Tick Value', 'Todays Range', 'Settlement Type',
'Base Symbol', '52 wk Range', 'Settlement Day', 'Point Value',
'1-Year Change', 'Last Rollover Day', 'Months'
])
result.at[0, 'Commodity Name'] = name
if path_:
for elements_ in path_:
element = elements_.xpath(".//span[@class='float_lang_base_1']")[0]
title_ = element.text_content()
if title_ == "Day's Range":
title_ = 'Todays Range'
if title_ in result.columns.tolist():
try:
result.at[0, title_] = float(
element.getnext().text_content().replace(',', ''))
continue
except:
pass
try:
text = element.getnext().text_content().strip()
result.at[0, title_] = datetime.strptime(
text, "%m/%d/%Y").strftime("%d/%m/%Y")
continue
except:
pass
try:
text = element.getnext().text_content().strip()
if text.__contains__('1 = '):
result.at[0, title_] = text.replace('1 = ', '')
continue
except:
pass
try:
value = element.getnext().text_content().strip()
if value.__contains__('K'):
value = float(value.replace('K', '').replace(',',
'')) * 1e3
elif value.__contains__('M'):
value = float(value.replace('M', '').replace(',',
'')) * 1e6
elif value.__contains__('B'):
value = float(value.replace('B', '').replace(',',
'')) * 1e9
elif value.__contains__('T'):
value = float(value.replace('T', '').replace(
',', '')) * 1e12
result.at[0, title_] = value
continue
except:
pass
result.replace({'N/A': None}, inplace=True)
if as_json is True:
json_ = result.iloc[0].to_dict()
return json_
elif as_json is False:
return result
else:
raise RuntimeError("ERR#0004: data retrieval error while scraping.")
