def _install_extension_media_internal(self, ext_class): """Install static media for an extension. Performs any installation necessary for an extension. If the extension has a modern static/ directory, they will be installed into :file:`{settings.STATIC_ROOT}/ext/`. """ if pkg_resources.resource_exists(ext_class.__module__, 'htdocs'): # This is an older extension that doesn't use the static file # support. Log a deprecation notice and then install the files. logger.error('The %s extension uses the deprecated "htdocs" ' 'directory for static files. This is no longer ' 'supported. It must be updated to use a "static" ' 'directory instead.', ext_class.info.name) ext_static_path = ext_class.info.installed_static_path ext_static_path_exists = os.path.exists(ext_static_path) if ext_static_path_exists: # Also get rid of the old static contents. shutil.rmtree(ext_static_path, ignore_errors=True) if pkg_resources.resource_exists(ext_class.__module__, 'static'): extracted_path = \ pkg_resources.resource_filename(ext_class.__module__, 'static') shutil.copytree(extracted_path, ext_static_path, symlinks=True)
def _iouImageBrowserSlot(self): """ Slot to open a file browser and select an IOU image. """ path = self.getIOUImage(self) if not path: return self.uiIOUPathLineEdit.clear() self.uiIOUPathLineEdit.setText(path) if "l2" in path: # set the default L2 base initial-config resource_name = "configs/iou_l2_base_initial-config.txt" if hasattr(sys, "frozen") and os.path.isfile(resource_name): self.uiInitialConfigLineEdit.setText(os.path.normpath(resource_name)) elif pkg_resources.resource_exists("gns3", resource_name): iou_base_config_path = pkg_resources.resource_filename("gns3", resource_name) self.uiInitialConfigLineEdit.setText(os.path.normpath(iou_base_config_path)) else: # set the default L3 base initial-config resource_name = "configs/iou_l3_base_initial-config.txt" if hasattr(sys, "frozen") and os.path.isfile(resource_name): self.uiInitialConfigLineEdit.setText(os.path.normpath(resource_name)) elif pkg_resources.resource_exists("gns3", resource_name): iou_base_config_path = pkg_resources.resource_filename("gns3", resource_name) self.uiInitialConfigLineEdit.setText(os.path.normpath(iou_base_config_path))
def resolve_main_template(): # Plone 5 main_template = os.path.join( 'browser', 'templates', 'main_template.pt') if pkg_resources.resource_exists('Products.CMFPlone', main_template): filename = pkg_resources.resource_filename('Products.CMFPlone', main_template) return ViewPageTemplateFile(filename) # Plone 4 with Sunburst sunburst_main_template = os.path.join( 'skins', 'sunburst_templates', 'main_template.pt') if pkg_resources.resource_exists('plonetheme.sunburst', sunburst_main_template): filename = pkg_resources.resource_filename('plonetheme.sunburst', sunburst_main_template) return ViewPageTemplateFile(filename) # Fallback skins_main_template = os.path.join( 'skins', 'plone_templates', 'main_template.pt') if pkg_resources.resource_exists('Products.CMFPlone', skins_main_template): filename = pkg_resources.resource_filename('Products.CMFPlone', skins_main_template) return ViewPageTemplateFile(filename)
def get_style_filepath(style, resolve_dependencies=True): """Get the full path of a style file. :param style: The name of the style (eg. 'apa') :param resolve_dependencies: If True, for dependent styles the independent style is parsed and it's path is returned. :returns: Filepath of a .csl style file. """ independent_style = os.path.join(independent_dir, '{0}.csl'.format(style)) if resource_exists(__name__, independent_style): return resource_filename(__name__, independent_style) dependent_style = os.path.join(dependent_dir, '{0}.csl'.format(style)) if resource_exists(__name__, dependent_style): style_path = resource_filename(__name__, dependent_style) if resolve_dependencies: inner_style = _resolve_dependent_style(style_path) inner_style = os.path.join(independent_dir, '{0}.csl'.format(inner_style)) if resource_exists(__name__, inner_style): return resource_filename(__name__, inner_style) else: raise StyleNotFoundError( 'The independent style {0} was not found.' .format(inner_style) ) return style_path raise StyleNotFoundError('The style {0} was not found.'.format(style))
def find_packaged_regressor(name): """ Find location of a regression method packaged with YATSM See :data:`packaged_regressions` for a list of available pre-packaged regressors Args: name (str): name of packaged regression object Returns: str: path to packaged regression method Raises: KeyError: raise KeyError if user specifies unknown regressor IOError: raise IOError if the packaged regressor cannot be found """ if name not in packaged_regressions: raise KeyError('Cannot load unknown packaged regressor %s' % name) path = pkg_resources.resource_filename(__name__, 'pickles') logger.debug('Checking data files in %s for packaged regressors' % path) if not pkg_resources.resource_exists(__name__, 'pickles'): raise IOError('Cannot find packaged regressors in %s. Did you install ' 'YATSM via setuptools?' % path) resource = os.path.join('pickles', name + '.pkl') if not pkg_resources.resource_exists(__name__, resource): raise IOError('Cannot find packaged regression method %s, but package ' 'directory exists. Check the contents of %s if possible' % (resource, path)) return pkg_resources.resource_filename(__name__, resource)
def resource_filename(name, pfx=None): """ Attempt to find and return the filename of the resource named by the first argument in the first location of: # as name in the current directory # as name in the `pfx` subdirectory of the current directory if provided # as name relative to the package # as pfx/name relative to the package The last two alternatives is used to locate resources distributed in the package. This includes certain XSLT and XSD files. :param name: The string name of a resource :param pfx: An optional prefix to use in searching for name """ if os.path.exists(name): with open(name) as fd: return fd.read() elif pfx and os.path.exists(os.path.join(pfx, name)): with open(os.path.join(pfx, name)) as fd: return fd.read() elif pkg_resources.resource_exists(__name__, name): return pkg_resources.resource_filename(__name__, name) elif pfx and pkg_resources.resource_exists(__name__, "%s/%s" % (pfx, name)): return pkg_resources.resource_filename(__name__, "%s/%s" % (pfx, name)) return None
def findEntryPointPlugins(allPlugins): # look for plugins enabled via setuptools `entry_points` for entry_point in iter_entry_points(group='girder.plugin'): # set defaults allPlugins[entry_point.name] = { 'name': entry_point.name, 'description': '', 'version': '', 'dependencies': set() } configJson = os.path.join('girder', 'plugin.json') configYml = os.path.join('girder', 'plugin.yml') data = {} try: if pkg_resources.resource_exists(entry_point.name, configJson): with pkg_resources.resource_stream( entry_point.name, configJson) as conf: try: data = json.load(codecs.getreader('utf8')(conf)) except ValueError: _recordPluginFailureInfo( plugin=entry_point.name, traceback=traceback.format_exc()) logprint.exception( 'ERROR: Plugin "%s": plugin.json is not valid ' 'JSON.' % entry_point.name) elif pkg_resources.resource_exists(entry_point.name, configYml): with pkg_resources.resource_stream( entry_point.name, configYml) as conf: try: data = yaml.safe_load(conf) except yaml.YAMLError: _recordPluginFailureInfo( plugin=entry_point.name, traceback=traceback.format_exc()) logprint.exception( 'ERROR: Plugin "%s": plugin.yml is not valid ' 'YAML.' % entry_point.name) except (ImportError, SystemError): # Fall through and just try to load the entry point below. If # there is still an error, we'll log it there. pass if data == {}: try: data = getattr(entry_point.load(), 'config', {}) except (ImportError, SystemError): # If the plugin failed to load via entrypoint, but is in the # plugins directory, it may still load. We mark and report the # failure; if it loads later, the failure will be cleared, but # the report is still desired. _recordPluginFailureInfo( plugin=entry_point.name, traceback=traceback.format_exc()) logprint.exception( 'ERROR: Plugin "%s": could not be loaded by entrypoint.' % entry_point.name) continue allPlugins[entry_point.name].update(data) allPlugins[entry_point.name]['dependencies'] = set( allPlugins[entry_point.name]['dependencies'])
def getSettings(self): """ Returns the settings set in this Wizard. :return: settings dict """ path = self.uiIOUImageLineEdit.text() if self.uiTypeComboBox.currentText() == "L2 image": # set the default L2 base initial-config resource_name = "configs/iou_l2_base_initial-config.txt" if hasattr(sys, "frozen") and os.path.isfile(resource_name): initial_config = os.path.normpath(resource_name) elif pkg_resources.resource_exists("gns3", resource_name): iou_base_config_path = pkg_resources.resource_filename("gns3", resource_name) initial_config = os.path.normpath(iou_base_config_path) default_symbol = ":/symbols/multilayer_switch.normal.svg" hover_symbol = ":/symbols/multilayer_switch.selected.svg" category = Node.switches else: # set the default L3 base initial-config resource_name = "configs/iou_l3_base_initial-config.txt" if hasattr(sys, "frozen") and os.path.isfile(resource_name): initial_config = os.path.normpath(resource_name) elif pkg_resources.resource_exists("gns3", resource_name): iou_base_config_path = pkg_resources.resource_filename("gns3", resource_name) initial_config = os.path.normpath(iou_base_config_path) default_symbol = ":/symbols/router.normal.svg" hover_symbol = ":/symbols/router.selected.svg" category = Node.routers if IOU.instance().settings()["use_local_server"] or self.uiLocalRadioButton.isChecked(): server = "local" elif self.uiRemoteRadioButton.isChecked(): if self.uiLoadBalanceCheckBox.isChecked(): server = next(iter(Servers.instance())) if not server: QtGui.QMessageBox.critical(self, "IOU device", "No remote server available!") return server = "{}:{}".format(server.host, server.port) else: server = self.uiRemoteServersComboBox.currentText() else: # Cloud is selected server = "cloud" settings = { "name": self.uiNameLineEdit.text(), "path": path, "image": os.path.basename(path), "initial_config": initial_config, "default_symbol": default_symbol, "category": category, "hover_symbol": hover_symbol, "server": server, } return settings
def _iouImageBrowserSlot(self): """ Slot to open a file browser and select an IOU image. """ #TODO: current directory for IOU image + filter? path, _ = QtGui.QFileDialog.getOpenFileNameAndFilter(self, "Select an IOU image", ".", "All files (*.*);;IOU image (*.bin *.image)", "IOU image (*.bin *.image)") if not path: return if not os.access(path, os.R_OK): QtGui.QMessageBox.critical(self, "IOU image", "Cannot read {}".format(path)) return try: with open(path, "rb") as f: # read the first 7 bytes of the file. elf_header_start = f.read(7) except OSError as e: QtGui.QMessageBox.critical(self, "IOU image", "Cannot read ELF magic number: {}".format(e)) return # file must start with the ELF magic number, be 32-bit, little endian and have an ELF version of 1 # normal IOS image are big endian! if elf_header_start != b'\x7fELF\x01\x01\x01': QtGui.QMessageBox.critical(self, "IOU image", "Sorry, this is not a valid IOU image!") return if not os.access(path, os.X_OK): QtGui.QMessageBox.critical(self, "IOU image", "{} is not executable".format(path)) return self.uiIOUPathLineEdit.clear() self.uiIOUPathLineEdit.setText(path) if "l2" in path: # set the default L2 base startup-config resource_name = "configs/iou_l2_base_startup-config.txt" if hasattr(sys, "frozen"): iou_base_config_path = os.path.join(os.path.dirname(sys.executable), resource_name) self.uiStartupConfigLineEdit.setText(os.path.relpath(os.path.normpath(iou_base_config_path))) elif pkg_resources.resource_exists("gns3", resource_name): iou_base_config_path = pkg_resources.resource_filename("gns3", resource_name) self.uiStartupConfigLineEdit.setText(os.path.relpath(os.path.normpath(iou_base_config_path))) else: # set the default L3 base startup-config resource_name = "configs/iou_l3_base_startup-config.txt" if hasattr(sys, "frozen"): iou_base_config_path = os.path.join(os.path.dirname(sys.executable), resource_name) self.uiStartupConfigLineEdit.setText(os.path.relpath(os.path.normpath(iou_base_config_path))) elif pkg_resources.resource_exists("gns3", resource_name): iou_base_config_path = pkg_resources.resource_filename("gns3", resource_name) self.uiStartupConfigLineEdit.setText(os.path.relpath(os.path.normpath(iou_base_config_path)))
def __getattr__(self, attr): file_loc = "data/" + self.directory + "/" + attr + ".json" dir_loc = "data/" + self.directory + "/" + attr if resource_exists(__name__, file_loc): return fetch_resource(file_loc) elif resource_exists(__name__, dir_loc) and \ resource_isdir(__name__, dir_loc): return CorpusLoader(self.directory + "/" + attr) else: raise AttributeError("no resource named " + attr)
def from_defaults(cls, namespace=None, name=None, *args, **kwargs): if namespace and name and resource_exists(namespace, name): _namespace = namespace _name = name elif (cls.resource_package and cls.resource_default and resource_exists(cls.resource_package, cls.resource_default)): _namespace = cls.resource_package _name = cls.resource_default else: log.warn('No class resource attributes and no namespace info, returning empty config') return cls(*args, **kwargs) return cls.from_pkg_resource(_namespace, _name, *args, **kwargs)
def __init__(self): # Check if the databases exist. if (not resource_exists(__name__, "data/prefixes.db") or not resource_exists(__name__, "data/words.db")): raise RuntimeError("can't find databases") # Generate the shelve objects. The '.db' suffix needs to be removed # from the filenames. self._prefix_db = shelve.open(resource_filename(__name__, "data/prefixes.db")[:-3]) self._word_db = shelve.open(resource_filename(__name__, "data/words.db")[:-3])
def __init__(self, name): self.name = "Sopra data from %s" % name import pkg_resources respath = 'data/Sopra/' if pkg_resources.resource_exists(__name__, respath+'%s.all' % name): raise NotImplementedError, "Multi-file luxpop data not implemented" elif pkg_resources.resource_exists(__name__, respath+'%s.nk' % name): sopra_file = pkg_resources.resource_stream(__name__, respath+'%s.nk' % (name)) else: raise RuntimeError, "Couldn't find Sopra datafile" SopraFile.__init__(self,sopra_file)
def resolve(self, system_url, public_id, context): """ Resolves URIs using the resource API """ log.debug("resolve SYSTEM URL' %s' for '%s'" % (system_url, public_id)) path = system_url.split("/") fn = path[len(path) - 1] if pkg_resources.resource_exists(__name__, fn): return self.resolve_file(pkg_resources.resource_stream(__name__, fn), context) elif pkg_resources.resource_exists(__name__, "schema/%s" % fn): return self.resolve_file(pkg_resources.resource_stream(__name__, "schema/%s" % fn), context) else: raise ValueError("Unable to locate %s" % fn)
def get_library(cls): if cls.library: return cls.library if resource_exists("mcviz.utils.svg.data", "texglyph.cache.bz2"): cls.library = loads(resource_string("mcviz.utils.svg.data", "texglyph.cache.bz2").decode("bz2")) elif resource_exists("mcviz.utils.svg.data", "texglyph.cache"): cls.library = loads(resource_string("mcviz.utils.svg.data", "texglyph.cache")) else: cls.make_library() with file(resource_filename("mcviz.utils.svg", "texglyph.cache"), "w") as f: f.write(dumps(cls.library, 2)) return cls.library
def loadTestsFromModule(self, module): """Return a suite of all tests cases contained in the given module If the module is a package, load tests from all the modules in it. If the module has an ``additional_tests`` function, call it and add the return value to the tests. """ tests = [] if module.__name__ != 'setuptools.tests.doctest': # ugh tests.append(TestLoader.loadTestsFromModule(self, module)) if hasattr(module, "additional_tests"): tests.append(module.additional_tests()) if hasattr(module, '__path__'): for file in resource_listdir(module.__name__, ''): if file.endswith('.py') and file != '__init__.py': submodule = module.__name__ + '.' + file[:-3] else: if resource_exists(module.__name__, file + '/__init__.py'): submodule = module.__name__+'.'+file else: continue tests.append(self.loadTestsFromName(submodule)) if len(tests) != 1: return self.suiteClass(tests) else: return tests[0] # don't create a nested suite for only one return
def get_icon(icon_id): "Returns an icon from the system theme or our fallback theme if required" fallback_path = os.path.join("fallback-theme", icon_id + ".png") if resource_exists(__name__, fallback_path): return QtGui.QIcon.fromTheme(icon_id, QtGui.QIcon(resource_filename(__name__, fallback_path))) else: return QtGui.QIcon.fromTheme(icon_id)
def make_css(minify=False): print("building css...") global _css_file data = [] loaded_modules = module.get_loaded_modules() for m in loaded_modules: if not pkg_resources.resource_exists(m, "css"): continue for fname in sorted(pkg_resources.resource_listdir(m, "css")): if not fname.endswith("css"): continue res = pkg_resources.resource_string(m, "css/"+fname).decode("utf-8") data.append(res) print(" %d css files loaded"%len(data)) if data: buf = ("\n".join(data)).encode("utf-8") m = hashlib.new("md5") m.update(buf) h = m.hexdigest()[:8] if not os.path.exists("static/css"): os.makedirs("static/css") open("static/css/netforce-%s.css"%h, "wb").write(buf) if minify: if not os.path.exists("static/css/netforce-%s-min.css"%h): print(" minifying css...") os.system("yui-compressor --type css static/css/netforce-%s.css > static/css/netforce-%s-min.css" %(h,h)) _css_file="netforce-%s-min.css"%h else: _css_file="netforce-%s.css"%h print(" => static/css/%s" % _css_file)
def get_file(path, mode='rb', source=SOURCE_ANY): """ Find and open the file at the given ``path``. This function first searches in the active profile's directory. If the file is not found there, and configuration is available, it attempts to use `pkg_resources <http://packages.python.org/distribute/pkg_resources.html>`_ to locate the file, otherwise it will search the root directory. If a file cannot be found, return ``None``. **Note:** ``mode`` is not used when getting a file from ``pkg_resources``. """ if os.path.isabs(path): if os.path.isfile(path): return open(path, mode) return None # Not an absolute path, so process it. ensure_paths() if source & SOURCE_PROFILE: p_path = os.path.join(profile_path, path) if os.path.isfile(p_path): return open(p_path, mode) if source & SOURCE_PKG_RESOURCES: if (package and pr and pr.resource_exists(package, path) and not pr.resource_isdir(package, path)): return pr.resource_stream(package, path) if source & SOURCE_ROOT: r_path = os.path.join(root_path, path) if os.path.isfile(r_path): return open(r_path, mode)
def __call__(self, context, request): if self.use_subpath: path_tuple = request.subpath else: path_tuple = traversal_path_info(request.environ['PATH_INFO']) path = _secure_path(path_tuple) if path is None: raise HTTPNotFound('Out of bounds: %s' % request.url) if self.package_name: # package resource resource_path ='%s/%s' % (self.docroot.rstrip('/'), path) if resource_isdir(self.package_name, resource_path): if not request.path_url.endswith('/'): self.add_slash_redirect(request) resource_path = '%s/%s' % (resource_path.rstrip('/'),self.index) if not resource_exists(self.package_name, resource_path): raise HTTPNotFound(request.url) filepath = resource_filename(self.package_name, resource_path) else: # filesystem file # os.path.normpath converts / to \ on windows filepath = normcase(normpath(join(self.norm_docroot, path))) if isdir(filepath): if not request.path_url.endswith('/'): self.add_slash_redirect(request) filepath = join(filepath, self.index) if not exists(filepath): raise HTTPNotFound(request.url) return FileResponse(filepath, request, self.cache_max_age)
def resource(name, pkg=None): if pkg is None: pkg = '%s.resources' % calling_package() resource = resource_filename(pkg, name) if not resource_exists(resource): raise OSError('Resource does not exist %s' % repr(resource)) return resource
def _unpack_resource(self, resource_path, resource_name, resource_executable): if not pkg_resources.resource_exists(__name__, resource_name): return if pkg_resources.resource_isdir(__name__, resource_name): self.__create_dir(resource_path) for f in pkg_resources.resource_listdir(__name__, resource_name): if f == "": # TODO(beng): Figure out why this happens continue # TODO: Handle executable resources in directory self._unpack_resource( os.path.join(resource_path, f), os.path.join(resource_name, f), False, ) else: with closable_named_temporary_file( prefix=resource_path + os.extsep ) as outf: outf.write(pkg_resources.resource_string(__name__, resource_name)) if resource_executable and hasattr(os, "fchmod"): st = os.fstat(outf.fileno()) os.fchmod(outf.fileno(), st.st_mode | stat.S_IXUSR) outf.close() shutil.copy(outf.name, resource_path)
def expand_macro(self, formatter, name, args): req = formatter.req chrome = Chrome(self.env) report = ReportModule(self.env) comma_splitter = re.compile(r'(?<!\\),') kwargs = {} for arg in comma_splitter.split(args): arg = arg.replace(r'\,', ',') m = re.match(r'\s*[^=]+=', arg) if m: kw = arg[:m.end() - 1].strip() value = arg[m.end():] if re.match(r'^\$[A-Z]*$', value): value = req.args.get(value[1:]) kwargs[kw] = value if value!= None else '' else: if re.match(r'^\$[A-Z]*$', arg): arg = req.args.get(arg[1:]) id = int(arg) req.args = kwargs req.args['page'] = '1' template, data, content_type = report._render_view(req, id) add_stylesheet(req, 'common/css/report.css') fullpath = '' if pkg_resources.resource_exists('wikireport', 'WikiReport.html'): fullpath = pkg_resources.resource_filename('wikireport', 'WikiReport.html') else: filepath = os.path.dirname(os.path.abspath( __file__ )) fullpath = os.path.join(filepath, 'WikiReport.html') return chrome.render_template(req, fullpath, data, None, fragment=True)
def _unpack_dir(self, resource_dir, dst_dir): if not pkg_resources.resource_exists(__name__, resource_dir): raise Exception( "Cannot unpack directory: {0} doesn't exist in the package".format( resource_dir ) ) if not pkg_resources.resource_isdir(__name__, resource_dir): raise Exception( "Cannot unpack directory: {0} is not a directory".format(resource_dir) ) self.__create_dir(dst_dir) if not os.path.exists(dst_dir): raise Exception( "Cannot unpack directory: cannot create directory {0}".format(dst_dir) ) for resource_file in pkg_resources.resource_listdir(__name__, resource_dir): resource_path = os.path.join(dst_dir, resource_file) if os.path.exists(resource_path): continue self._unpack_resource( resource_path, "/".join((resource_dir, resource_file)), False )
def _load_ttbl(lang): language, territory = _split_language(lang) language_resource = 'localedata/%s.ttbl' % language full_resource = 'localedata/%s_%s.ttbl' % (language, territory) fallback_resource = 'localedata/%s_%s.ttbl' % (language, language.upper()) candidates = [ full_resource + '.bz2', full_resource, language_resource + '.bz2', language_resource, fallback_resource + '.bz2', fallback_resource, ] for c in candidates: if pkg_resources.resource_exists(__name__, c): with pkg_resources.resource_stream(__name__, c) as f: buf = f.read() if c.endswith('.bz2'): buf = bz2.decompress(buf) return pickle.loads(buf) raise LanguageNotFoundError('Could not find translation table for %s' % lang)
def run_after(command): """ Copy the free-cad macros/modules in the macro directory. @param command: setuptools command instance, either an InstallCommand or DevelopCommand. """ req = pkg_resources.Requirement.parse(command.distribution.get_name()) if pkg_resources.resource_exists(req, "freecad"): # get freecad macro dir macrodir = None devnull = open(os.devnull, 'w') for executable in ["freecadcmd", "FreeCADCmd"]: try: macrodir = subprocess.check_output([executable, "--get-config", "UserAppData"], stderr=devnull) break except: pass if macrodir is None: print "FreeCAD macro directory not found, please read the installation instructions !" return macrodir = macrodir.strip() print "Using macrodir:", macrodir # recursively copy files/directories to the freecad macro dir try: copy_macros(req, "freecad", macrodir) except Exception, ex: print "Error:", ex print "Failed copying macros, please read the installation instructions !"
def getResource(identifier, pkgname=__name__): """ Acquire a readable object for a given package name and identifier. An IOError will be raised if the resource can not be found. For example: mydata = getResource('mypkgdata.jpg').read() Note that the package name must be fully qualified, if given, such that it would be found in sys.modules. In some cases, getResource will return a real file object. In that case, it may be useful to use its name attribute to get the path rather than use it as a file-like object. For example, you may be handing data off to a C API. """ if resource_exists(pkgname, identifier): return resource_stream(pkgname, identifier) mod = sys.modules[pkgname] fn = getattr(mod, '__file__', None) if fn is None: raise IOError("%s has no __file__!" % repr(mod)) path = os.path.join(os.path.dirname(fn), identifier) loader = getattr(mod, '__loader__', None) if loader is not None: try: data = loader.get_data(path) except IOError: pass else: return BytesIO(data) return open(os.path.normpath(path), 'rb')
def __init__(self): """Imports all plugin modules based on platform""" for modname in pkg_resources.resource_listdir(__name__,'plugins'): if pkg_resources.resource_exists(__name__, 'plugins/%s/__init__.py' % modname): plugin = __import__('plugins.%s' % modname, globals(), locals(), ['']).get_plugin() if plugin.platform == 'All' or plugin.platform == howl.platform: self[modname] = plugin
def _get_template(self, template_name): """Given a template name, attempt to resolve it and return the contents as a String.""" if pkg_resources.resource_exists(__name__, 'templates/%s' % template_name): # loading from our recipe egg return pkg_resources.resource_string(__name__, 'templates/%s' % template_name) else: # see if the user specified a templates directory template_dir = self.options.get('template_dir', os.path.join(self.buildout['buildout']['directory'], 'templates')) if os.path.exists(os.path.join(template_dir, template_name)): return file(os.path.join(template_dir, template_name)).read() # unable to find template, throw an error logging.getLogger(self.name).error("Template %s does not exist." % template_name) raise zc.buildout.UserError( "The specified template, %s, does not exist" % template_name)
def loadTestsFromModule(self, module): """Load unit test (skip 'interop' package). Hacked from the version in 'setuptools.command.test.ScanningLoader'. """ tests = [] tests.append(TestLoader.loadTestsFromModule(self,module)) if hasattr(module, '__path__'): for file in resource_listdir(module.__name__, ''): if file == 'interop': # These tests require installing a bunch of extra # code: see 'src/soaplib/test/README'. continue if file.endswith('.py') and file != '__init__.py': submodule = module.__name__ + '.' + file[:-3] else: if resource_exists( module.__name__, file + '/__init__.py' ): submodule = module.__name__ + '.' + file else: continue tests.append(self.loadTestsFromName(submodule)) return self.suiteClass(tests)
def get_currency_cross_information(currency_cross, as_json=False): """ This function retrieves fundamental financial information from the specified currency cross. The retrieved information from the currency cross can be valuable as it is additional information that can be used combined with OHLC values, so to determine financial insights from the company which holds the specified currency cross. Args: currency_cross (:obj:`str`): name of the currency_cross to retrieve recent historical data from. as_json (:obj:`bool`, optional): optional argument to determine the format of the output data (:obj:`dict` or :obj:`json`). Returns: :obj:`pandas.DataFrame` or :obj:`dict`- currency cross_information: The resulting :obj:`pandas.DataFrame` contains the information fields retrieved from Investing.com from the specified currency cross; it can also be returned as a :obj:`dict`, if argument `as_json=True`. If any of the information fields could not be retrieved, that field/s will be filled with None values. If the retrieval process succeeded, the resulting :obj:`dict` will look like:: currency_cross_information = { "1-Year Change": "- 1.61%", "52 wk Range": "1.0879 - 1.1572", "Ask": 1.1144, "Bid": 1.114, "Currency Cross": "EUR/USD", "Open": 1.1121, "Prev. Close": 1.1119, "Todays Range": "1.1123 - 1.1159" } Raises: ValueError: raised if any of the introduced arguments is not valid or errored. FileNotFoundError: raised if `currency_crosses.csv` file was not found. IOError: raised if `currency_crosses.csv` file is empty or errored. RuntimeError: raised if scraping process failed while running. ConnectionError: raised if the connection to Investing.com errored (did not return HTTP 200) """ if not currency_cross: raise ValueError("ERR#0052: currency_cross param is mandatory and should be a str.") if not isinstance(currency_cross, str): raise ValueError("ERR#0052: currency_cross param is mandatory and should be a str.") if not isinstance(as_json, bool): raise ValueError("ERR#0002: as_json argument can just be True or False, bool type.") resource_package = 'investpy' resource_path = '/'.join(('resources', 'currency_crosses.csv')) if pkg_resources.resource_exists(resource_package, resource_path): crosses = pd.read_csv(pkg_resources.resource_filename(resource_package, resource_path)) else: raise FileNotFoundError("ERR#0060: currency_crosses file not found or errored.") if crosses is None: raise IOError("ERR#0050: currency_crosses not found or unable to retrieve.") currency_cross = currency_cross.strip() currency_cross = currency_cross.lower() if unidecode(currency_cross) not in [unidecode(value.lower()) for value in crosses['name'].tolist()]: raise RuntimeError("ERR#0054: the introduced currency_cross " + str(currency_cross) + " does not exists.") name = crosses.loc[(crosses['name'].str.lower() == currency_cross).idxmax(), 'name'] tag = crosses.loc[(crosses['name'].str.lower() == currency_cross).idxmax(), 'tag'] url = "https://www.investing.com/currencies/" + tag head = { "User-Agent": random_user_agent(), "X-Requested-With": "XMLHttpRequest", "Accept": "text/html", "Accept-Encoding": "gzip, deflate, br", "Connection": "keep-alive", } req = requests.get(url, headers=head) if req.status_code != 200: raise ConnectionError("ERR#0015: error " + str(req.status_code) + ", try again later.") root_ = fromstring(req.text) path_ = root_.xpath("//div[contains(@class, 'overviewDataTable')]/div") result = pd.DataFrame(columns=['Currency Cross', 'Prev. Close', 'Bid', 'Todays Range', 'Open', 'Ask', '52 wk Range', '1-Year Change']) result.at[0, 'Currency Cross'] = name if path_: for elements_ in path_: element = elements_.xpath(".//span[@class='float_lang_base_1']")[0] title_ = element.text_content() if title_ == "Day's Range": title_ = 'Todays Range' if title_ in result.columns.tolist(): try: result.at[0, title_] = float(element.getnext().text_content().replace(',', '')) continue except: pass try: text = element.getnext().text_content().strip() result.at[0, title_] = datetime.strptime(text, "%b %d, %Y").strftime("%d/%m/%Y") continue except: pass try: value = element.getnext().text_content().strip() if value.__contains__('K'): value = float(value.replace('K', '').replace(',', '')) * 1e3 elif value.__contains__('M'): value = float(value.replace('M', '').replace(',', '')) * 1e6 elif value.__contains__('B'): value = float(value.replace('B', '').replace(',', '')) * 1e9 elif value.__contains__('T'): value = float(value.replace('T', '').replace(',', '')) * 1e12 result.at[0, title_] = value continue except: pass result.replace({'N/A': None}, inplace=True) if as_json is True: json_ = result.iloc[0].to_dict() return json_ elif as_json is False: return result else: raise RuntimeError("ERR#0004: data retrieval error while scraping.")
def get_commodities_overview(group, as_json=False, n_results=100): """ This function retrieves an overview containing all the real time data available for the main commodities from every commodity group (metals, softs, meats, energy and grains), such as the names, symbols, current value, etc. as indexed in Investing.com. So on, the main usage of this function is to get an overview on the main commodities from a group, so to get a general view. Note that since this function is retrieving a lot of information at once, by default just the overview of the Top 100 commodities is being retrieved, but an additional parameter called n_results can be specified so to retrieve N results. Anyways, note that in commodities case, there are just a few ones available. Args: group (:obj:`str`): name of the commodity group to retrieve an overview from. as_json (:obj:`bool`, optional): optional argument to determine the format of the output data (:obj:`pandas.DataFrame` or :obj:`json`). n_results (:obj:`int`, optional): number of results to be displayed on the overview table (0-1000). Returns: :obj:`pandas.DataFrame` - commodities_overview: The resulting :obj:`pandas.DataFrame` contains all the data available in Investing.com of the main commodities from a commodity group in order to get an overview of it. If the retrieval process succeeded, the resulting :obj:`pandas.DataFrame` should look like:: country | name | last | last_close | high | low | change | change_percentage | currency --------|------|------|------------|------|-----|--------|-------------------|---------- xxxxxxx | xxxx | xxxx | xxxxxxxxxx | xxxx | xxx | xxxxxx | xxxxxxxxxxxxxxxxx | xxxxxxxx Raises: ValueError: raised if any of the introduced arguments errored. FileNotFoundError: raised if `commodities.csv` file is missing. IOError: raised if data could not be retrieved due to file error. RuntimeError: raised either if the introduced group does not match any of the listed ones or if no overview results could be retrieved from Investing.com. ConnectionError: raised if GET requests does not return 200 status code. """ if group is None: raise ValueError( "ERR#0090: group can not be None, it should be a str.") if group is not None and not isinstance(group, str): raise ValueError( "ERR#0090: group can not be None, it should be a str.") if not isinstance(as_json, bool): raise ValueError( "ERR#0002: as_json argument can just be True or False, bool type.") if not isinstance(n_results, int): raise ValueError( "ERR#0089: n_results argument should be an integer between 1 and 1000." ) if 1 > n_results or n_results > 1000: raise ValueError( "ERR#0089: n_results argument should be an integer between 1 and 1000." ) resource_package = 'investpy' resource_path = '/'.join(('resources', 'commodities.csv')) if pkg_resources.resource_exists(resource_package, resource_path): commodities = pd.read_csv( pkg_resources.resource_filename(resource_package, resource_path)) else: raise FileNotFoundError( "ERR#0075: commodities file not found or errored.") if commodities is None: raise IOError("ERR#0076: commodities not found or unable to retrieve.") group = unidecode(group.lower()) if group not in get_commodity_groups(): raise RuntimeError( 'ERR#0091: specified commodity group value is not valid.') commodities = commodities[commodities['group'] == group] head = { "User-Agent": random_user_agent(), "X-Requested-With": "XMLHttpRequest", "Accept": "text/html", "Accept-Encoding": "gzip, deflate, br", "Connection": "keep-alive", } url = "https://www.investing.com/commodities/" + group req = requests.get(url, headers=head) if req.status_code != 200: raise ConnectionError("ERR#0015: error " + str(req.status_code) + ", try again later.") root_ = fromstring(req.text) table = root_.xpath(".//table[@id='cross_rate_1']/tbody/tr") results = list() if len(table) > 0: for row in table[:n_results]: id_ = row.get('id').replace('pair_', '') country_check = row.xpath(".//td[@class='flag']/span")[0].get( 'title').lower() name = row.xpath( ".//td[contains(@class, 'elp')]/a")[0].text_content().strip() pid = 'pid-' + id_ last = row.xpath(".//td[@class='" + pid + "-last']")[0].text_content() last_close = row.xpath(".//td[@class='" + pid + "-last_close']")[0].text_content() high = row.xpath(".//td[@class='" + pid + "-high']")[0].text_content() low = row.xpath(".//td[@class='" + pid + "-low']")[0].text_content() pc = row.xpath(".//td[contains(@class, '" + pid + "-pc')]")[0].text_content() pcp = row.xpath(".//td[contains(@class, '" + pid + "-pcp')]")[0].text_content() data = { "country": country_check if country_check != '' else None, "name": name, "last": float(last.replace(',', '')), "last_close": float(last_close.replace(',', '')), "high": float(high.replace(',', '')), "low": float(low.replace(',', '')), "change": pc, "change_percentage": pcp, "currency": commodities.loc[( (commodities['name'] == name) & (commodities['country'] == country_check)).idxmax(), 'currency'] if country_check != '' else commodities.loc[(commodities['name'] == name).idxmax(), 'currency'] } results.append(data) else: raise RuntimeError( "ERR#0092: no data found while retrieving the overview from Investing.com" ) df = pd.DataFrame(results) if as_json: return json.loads(df.to_json(orient='records')) else: return df
def _get_available_locales(self): return sorted( locale for locale in translation.get_available_locales() if resource_exists('trac', 'locale/%s/LC_MESSAGES/messages.mo' % locale))
def test_resource_exists(module_name, resource_name): assert pkg_resources.resource_exists(module_name, resource_name) is True
def main(argv=None): if argv is None: argv = sys.argv usage = "usage: %prog [options] [workflow_file]" parser = optparse.OptionParser(usage=usage) parser.add_option("--no-discovery", action="store_true", help="Don't run widget discovery " "(use full cache instead)") parser.add_option("--force-discovery", action="store_true", help="Force full widget discovery " "(invalidate cache)") parser.add_option("--clear-widget-settings", action="store_true", help="Remove stored widget setting") parser.add_option("--no-welcome", action="store_true", help="Don't show welcome dialog.") parser.add_option("--no-splash", action="store_true", help="Don't show splash screen.") parser.add_option("-l", "--log-level", help="Logging level (0, 1, 2, 3, 4)", type="int", default=1) parser.add_option("--style", help="QStyle to use", type="str", default=None) parser.add_option("--stylesheet", help="Application level CSS style sheet to use", type="str", default="orange.qss") parser.add_option("--qt", help="Additional arguments for QApplication", type="str", default=None) (options, args) = parser.parse_args(argv[1:]) levels = [ logging.CRITICAL, logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG ] # Fix streams before configuring logging (otherwise it will store # and write to the old file descriptors) fix_win_pythonw_std_stream() # Try to fix fonts on OSX Mavericks fix_osx_10_9_private_font() # File handler should always be at least INFO level so we need # the application root level to be at least at INFO. root_level = min(levels[options.log_level], logging.INFO) rootlogger = logging.getLogger(canvas.__name__) rootlogger.setLevel(root_level) # Initialize SQL query and execution time logger (in SqlTable) sql_level = min(levels[options.log_level], logging.INFO) make_sql_logger(sql_level) # Standard output stream handler at the requested level stream_hander = logging.StreamHandler() stream_hander.setLevel(level=levels[options.log_level]) rootlogger.addHandler(stream_hander) log.info("Starting 'Orange Canvas' application.") qt_argv = argv[:1] if options.style is not None: qt_argv += ["-style", options.style] if options.qt is not None: qt_argv += shlex.split(options.qt) qt_argv += args log.debug("Starting CanvasApplicaiton with argv = %r.", qt_argv) app = CanvasApplication(qt_argv) # NOTE: config.init() must be called after the QApplication constructor config.init() clear_settings_flag = os.path.join(config.widget_settings_dir(), "DELETE_ON_START") if options.clear_widget_settings or \ os.path.isfile(clear_settings_flag): log.info("Clearing widget settings") shutil.rmtree(config.widget_settings_dir(), ignore_errors=True) file_handler = logging.FileHandler(filename=os.path.join( config.log_dir(), "canvas.log"), mode="w") file_handler.setLevel(root_level) rootlogger.addHandler(file_handler) # intercept any QFileOpenEvent requests until the main window is # fully initialized. # NOTE: The QApplication must have the executable ($0) and filename # arguments passed in argv otherwise the FileOpen events are # triggered for them (this is done by Cocoa, but QApplicaiton filters # them out if passed in argv) open_requests = [] def onrequest(url): log.info("Received an file open request %s", url) open_requests.append(url) app.fileOpenRequest.connect(onrequest) settings = QSettings() stylesheet = options.stylesheet stylesheet_string = None if stylesheet != "none": if os.path.isfile(stylesheet): with open(stylesheet, "r") as f: stylesheet_string = f.read() else: if not os.path.splitext(stylesheet)[1]: # no extension stylesheet = os.path.extsep.join([stylesheet, "qss"]) pkg_name = canvas.__name__ resource = "styles/" + stylesheet if pkg_resources.resource_exists(pkg_name, resource): stylesheet_string = \ pkg_resources.resource_string(pkg_name, resource).decode() base = pkg_resources.resource_filename(pkg_name, "styles") pattern = re.compile( r"^\s@([a-zA-Z0-9_]+?)\s*:\s*([a-zA-Z0-9_/]+?);\s*$", flags=re.MULTILINE) matches = pattern.findall(stylesheet_string) for prefix, search_path in matches: QDir.addSearchPath(prefix, os.path.join(base, search_path)) log.info("Adding search path %r for prefix, %r", search_path, prefix) stylesheet_string = pattern.sub("", stylesheet_string) else: log.info("%r style sheet not found.", stylesheet) # Add the default canvas_icons search path dirpath = os.path.abspath(os.path.dirname(canvas.__file__)) QDir.addSearchPath("canvas_icons", os.path.join(dirpath, "icons")) canvas_window = CanvasMainWindow() canvas_window.setWindowIcon(config.application_icon()) if stylesheet_string is not None: canvas_window.setStyleSheet(stylesheet_string) if not options.force_discovery: reg_cache = cache.registry_cache() else: reg_cache = None widget_discovery = qt.QtWidgetDiscovery(cached_descriptions=reg_cache) widget_registry = qt.QtWidgetRegistry() widget_discovery.found_category.connect(widget_registry.register_category) widget_discovery.found_widget.connect(widget_registry.register_widget) want_splash = \ settings.value("startup/show-splash-screen", True, type=bool) and \ not options.no_splash if want_splash: pm, rect = config.splash_screen() splash_screen = SplashScreen(pixmap=pm, textRect=rect) splash_screen.setFont(QFont("Helvetica", 12)) color = QColor("#FFD39F") def show_message(message): splash_screen.showMessage(message, color=color) widget_discovery.discovery_start.connect(splash_screen.show) widget_discovery.discovery_process.connect(show_message) widget_discovery.discovery_finished.connect(splash_screen.hide) log.info("Running widget discovery process.") cache_filename = os.path.join(cache_dir(), "widget-registry.pck") if options.no_discovery: with open(cache_filename, "rb") as f: widget_registry = pickle.load(f) widget_registry = qt.QtWidgetRegistry(widget_registry) else: widget_discovery.run(config.widgets_entry_points()) # Store cached descriptions cache.save_registry_cache(widget_discovery.cached_descriptions) with open(cache_filename, "wb") as f: pickle.dump(WidgetRegistry(widget_registry), f) set_global_registry(widget_registry) canvas_window.set_widget_registry(widget_registry) canvas_window.show() canvas_window.raise_() want_welcome = \ settings.value("startup/show-welcome-screen", True, type=bool) \ and not options.no_welcome # Process events to make sure the canvas_window layout has # a chance to activate (the welcome dialog is modal and will # block the event queue, plus we need a chance to receive open file # signals when running without a splash screen) app.processEvents() app.fileOpenRequest.connect(canvas_window.open_scheme_file) if want_welcome and not args and not open_requests: canvas_window.welcome_dialog() elif args: log.info("Loading a scheme from the command line argument %r", args[0]) canvas_window.load_scheme(args[0]) elif open_requests: log.info("Loading a scheme from an `QFileOpenEvent` for %r", open_requests[-1]) canvas_window.load_scheme(open_requests[-1].toLocalFile()) # local references prevent destruction _ = show_survey() __ = check_for_updates() # Tee stdout and stderr into Output dock log_view = canvas_window.log_view() stdout = TextStream() stdout.stream.connect(log_view.write) if sys.stdout: stdout.stream.connect(sys.stdout.write) stdout.flushed.connect(sys.stdout.flush) stderr = TextStream() error_writer = log_view.formated(color=Qt.red) stderr.stream.connect(error_writer.write) if sys.stderr: stderr.stream.connect(sys.stderr.write) stderr.flushed.connect(sys.stderr.flush) log.info("Entering main event loop.") try: with patch('sys.excepthook', ExceptHook(stream=stderr, canvas=canvas_window, handledException=handle_exception)),\ patch('sys.stderr', stderr),\ patch('sys.stdout', stdout): status = app.exec_() except BaseException: log.error("Error in main event loop.", exc_info=True) canvas_window.deleteLater() app.processEvents() app.flush() del canvas_window # Collect any cycles before deleting the QApplication instance gc.collect() del app return status
def get_commodity_historical_data(commodity, from_date, to_date, country=None, as_json=False, order='ascending', interval='Daily'): """ This function retrieves historical data from the introduced commodity from Investing.com. So on, the historical data of the introduced commodity in the specified date range will be retrieved and returned as a :obj:`pandas.DataFrame` if the parameters are valid and the request to Investing.com succeeds. Note that additionally some optional parameters can be specified: as_json and order, which let the user decide if the data is going to be returned as a :obj:`json` or not, and if the historical data is going to be ordered ascending or descending (where the index is the date), respectively. Args: commodity (:obj:`str`): name of the commodity to retrieve recent data from. from_date (:obj:`str`): date formatted as `dd/mm/yyyy`, since when data is going to be retrieved. to_date (:obj:`str`): date formatted as `dd/mm/yyyy`, until when data is going to be retrieved. country (:obj:`str`, optional): name of the country to retrieve the commodity data from (if there is more than one country that provides data from the same commodity). as_json (:obj:`bool`, optional): to determine the format of the output data, either a :obj:`pandas.DataFrame` if False and a :obj:`json` if True. order (:obj:`str`, optional): to define the order of the retrieved data which can either be ascending or descending. interval (:obj:`str`, optional): value to define the historical data interval to retrieve, by default `Daily`, but it can also be `Weekly` or `Monthly`. Returns: :obj:`pandas.DataFrame` or :obj:`json`: The function returns a either a :obj:`pandas.DataFrame` or a :obj:`json` file containing the retrieved historical data of the specified commodity. So on, the resulting dataframe contains the open, high, low and close values for the selected commodity on market days and the currency in which those values are presented. The returned data is case we use default arguments will look like:: Date || Open | High | Low | Close | Volume | Currency -----||------|------|-----|-------|--------|---------- xxxx || xxxx | xxxx | xxx | xxxxx | xxxxxx | xxxxxxxx but in case that as_json parameter was defined as True, then the output will be:: { name: name, historical: [ { date: 'dd/mm/yyyy', open: x, high: x, low: x, close: x, volume: x, currency: x }, ... ] } Raises: ValueError: raised whenever any of the introduced arguments is not valid or errored. IOError: raised if commodities object/file was not found or unable to retrieve. RuntimeError: raised if the introduced commodity was not found or did not match any of the existing ones. ConnectionError: raised if connection to Investing.com could not be established. IndexError: raised if commodity historical data was unavailable or not found in Investing.com. Examples: >>> data = investpy.get_historical_data(commodity='gold', from_date='01/01/2018', to_date='01/01/2019') >>> data.head() Open High Low Close Volume Currency Date 2018-01-01 1305.8 1309.7 1304.6 1308.7 0 USD 2018-01-02 1370.5 1370.5 1370.5 1370.5 97 USD 2018-01-03 1372.0 1372.0 1369.0 1374.2 22 USD 2018-01-04 1363.4 1375.6 1362.7 1377.4 13 USD 2018-01-05 1377.8 1377.8 1377.8 1378.4 10 USD """ if not commodity: raise ValueError( "ERR#0078: commodity parameter is mandatory and must be a valid commodity name." ) if not isinstance(commodity, str): raise ValueError( "ERR#0078: commodity parameter is mandatory and must be a valid commodity name." ) if country is not None and not isinstance(country, str): raise ValueError("ERR#0025: specified country value not valid.") if not isinstance(as_json, bool): raise ValueError( "ERR#0002: as_json argument can just be True or False, bool type.") if order not in ['ascending', 'asc', 'descending', 'desc']: raise ValueError( "ERR#0003: order argument can just be ascending (asc) or descending (desc), str type." ) if not interval: raise ValueError( "ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'." ) if not isinstance(interval, str): raise ValueError( "ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'." ) interval = interval.lower() if interval not in ['daily', 'weekly', 'monthly']: raise ValueError( "ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'." ) try: datetime.strptime(from_date, '%d/%m/%Y') except ValueError: raise ValueError( "ERR#0011: incorrect from_date date format, it should be 'dd/mm/yyyy'." ) try: datetime.strptime(to_date, '%d/%m/%Y') except ValueError: raise ValueError( "ERR#0012: incorrect to_date format, it should be 'dd/mm/yyyy'.") start_date = datetime.strptime(from_date, '%d/%m/%Y') end_date = datetime.strptime(to_date, '%d/%m/%Y') if start_date >= end_date: raise ValueError( "ERR#0032: to_date should be greater than from_date, both formatted as 'dd/mm/yyyy'." ) date_interval = { 'intervals': [], } flag = True while flag is True: diff = end_date.year - start_date.year if diff > 19: obj = { 'start': start_date.strftime('%m/%d/%Y'), 'end': start_date.replace(year=start_date.year + 19).strftime('%m/%d/%Y'), } date_interval['intervals'].append(obj) start_date = start_date.replace(year=start_date.year + 19, day=start_date.day + 1) else: obj = { 'start': start_date.strftime('%m/%d/%Y'), 'end': end_date.strftime('%m/%d/%Y'), } date_interval['intervals'].append(obj) flag = False interval_limit = len(date_interval['intervals']) interval_counter = 0 data_flag = False resource_package = 'investpy' resource_path = '/'.join(('resources', 'commodities.csv')) if pkg_resources.resource_exists(resource_package, resource_path): commodities = pd.read_csv( pkg_resources.resource_filename(resource_package, resource_path)) else: raise FileNotFoundError( "ERR#0075: commodities file not found or errored.") if commodities is None: raise IOError("ERR#0076: commodities not found or unable to retrieve.") commodity = commodity.strip() commodity = commodity.lower() if unidecode(commodity) not in [ unidecode(value.lower()) for value in commodities['name'].tolist() ]: raise RuntimeError("ERR#0079: commodity " + commodity + " not found, check if it is correct.") if country is None: found_commodities = commodities[commodities['name'].str.lower() == commodity] if len(found_commodities) > 1: msg = "Note that the displayed commodity data can differ depending on the country. " \ "If you want to retrieve " + commodity + " data from either " + \ " or ".join(found_commodities['country'].tolist()) + ", specify the country parameter." warnings.warn(msg, Warning) del found_commodities else: if unidecode(country.lower()) not in commodities['country'].unique( ).tolist(): raise RuntimeError("ERR#0034: country " + country.lower() + " not found, check if it is correct.") commodities = commodities[commodities['country'] == unidecode( country.lower())] full_name = commodities.loc[( commodities['name'].str.lower() == commodity).idxmax(), 'full_name'] id_ = commodities.loc[( commodities['name'].str.lower() == commodity).idxmax(), 'id'] name = commodities.loc[( commodities['name'].str.lower() == commodity).idxmax(), 'name'] currency = commodities.loc[( commodities['name'].str.lower() == commodity).idxmax(), 'currency'] header = full_name + ' Historical Data' final = list() for index in range(len(date_interval['intervals'])): interval_counter += 1 params = { "curr_id": id_, "smlID": str(randint(1000000, 99999999)), "header": header, "st_date": date_interval['intervals'][index]['start'], "end_date": date_interval['intervals'][index]['end'], "interval_sec": interval.capitalize(), "sort_col": "date", "sort_ord": "DESC", "action": "historical_data" } head = { "User-Agent": random_user_agent(), "X-Requested-With": "XMLHttpRequest", "Accept": "text/html", "Accept-Encoding": "gzip, deflate, br", "Connection": "keep-alive", } url = "https://www.investing.com/instruments/HistoricalDataAjax" req = requests.post(url, headers=head, data=params) if req.status_code != 200: raise ConnectionError("ERR#0015: error " + str(req.status_code) + ", try again later.") if not req.text: continue root_ = fromstring(req.text) path_ = root_.xpath(".//table[@id='curr_table']/tbody/tr") result = list() if path_: for elements_ in path_: if elements_.xpath( ".//td")[0].text_content() == 'No results found': if interval_counter < interval_limit: data_flag = False else: raise IndexError( "ERR#0080: commodity information unavailable or not found." ) else: data_flag = True info = [] for nested_ in elements_.xpath(".//td"): info.append(nested_.get('data-real-value')) if data_flag is True: commodity_date = datetime.strptime( str( datetime.fromtimestamp( int(info[0]), tz=pytz.timezone('GMT')).date()), '%Y-%m-%d') commodity_close = float(info[1].replace(',', '')) commodity_open = float(info[2].replace(',', '')) commodity_high = float(info[3].replace(',', '')) commodity_low = float(info[4].replace(',', '')) commodity_volume = int(info[5]) result.insert( len(result), Data(commodity_date, commodity_open, commodity_high, commodity_low, commodity_close, commodity_volume, currency, None)) if data_flag is True: if order in ['ascending', 'asc']: result = result[::-1] elif order in ['descending', 'desc']: result = result if as_json is True: json_list = [value.commodity_as_json() for value in result] final.append(json_list) elif as_json is False: df = pd.DataFrame.from_records( [value.commodity_to_dict() for value in result]) df.set_index('Date', inplace=True) final.append(df) else: raise RuntimeError( "ERR#0004: data retrieval error while scraping.") if order in ['descending', 'desc']: final.reverse() if as_json is True: json_ = { 'name': name, 'historical': [value for json_list in final for value in json_list] } return json.dumps(json_, sort_keys=False) elif as_json is False: return pd.concat(final)
def search_commodities(by, value): """ This function searches commodities by the introduced value for the specified field. This means that this function is going to search if there is a value that matches the introduced one for the specified field which is the `commodities.csv` column name to search in. Available fields to search commodities are 'name', 'full_name' and 'title'. Args: by (:obj:`str`): name of the field to search for, which is the column name which can be: ''name', 'full_name' or 'title'. value (:obj:`str`): value of the field to search for, which is the value that is going to be searched. Returns: :obj:`pandas.DataFrame` - search_result: The resulting :obj:`pandas.DataFrame` contains the search results from the given query, which is any match of the specified value in the specified field. If there are no results for the given query, an error will be raised, but otherwise the resulting :obj:`pandas.DataFrame` will contain all the available commodities that match the introduced query. Raises: ValueError: raised if any of the introduced parameters is not valid or errored. FileNotFoundError: raised if `commodities.csv` file is missing. IOError: raised if data could not be retrieved due to file error. RuntimeError: raised if no results were found for the introduced value in the introduced field. """ if not by: raise ValueError( 'ERR#0006: the introduced field to search is mandatory and should be a str.' ) if not isinstance(by, str): raise ValueError( 'ERR#0006: the introduced field to search is mandatory and should be a str.' ) if not value: raise ValueError( 'ERR#0017: the introduced value to search is mandatory and should be a str.' ) if not isinstance(value, str): raise ValueError( 'ERR#0017: the introduced value to search is mandatory and should be a str.' ) resource_package = 'investpy' resource_path = '/'.join(('resources', 'commodities.csv')) if pkg_resources.resource_exists(resource_package, resource_path): commodities = pd.read_csv( pkg_resources.resource_filename(resource_package, resource_path)) else: raise FileNotFoundError( "ERR#0075: commodities file not found or errored.") if commodities is None: raise IOError("ERR#0076: commodities not found or unable to retrieve.") commodities.drop(columns=['tag', 'id'], inplace=True) available_search_fields = commodities.columns.tolist() if isinstance(by, str) and by not in available_search_fields: raise ValueError( 'ERR#0026: the introduced field to search can either just be ' + ' or '.join(available_search_fields)) commodities['matches'] = commodities[by].str.contains(value, case=False) search_result = commodities.loc[commodities['matches'] == True].copy() if len(search_result) == 0: raise RuntimeError( 'ERR#0043: no results were found for the introduced ' + str(by) + '.') search_result.drop(columns=['matches'], inplace=True) search_result.reset_index(drop=True, inplace=True) return search_result
def etfs_as_dict(country=None, columns=None, as_json=False): """ This function retrieves all the available etfs and returns a dictionary with the specified columns. Available columns are: 'id', 'name', 'symbol' and 'tag' All the available etfs can be found at: https://es.investing.com/etfs/spain-etfs Returns ------- :returns a dictionary that contains all the available etf values specified in the columns """ if country is not None and not isinstance(country, str): raise IOError("ERR#0025: specified country value not valid.") if columns is None: columns = ['id', 'name', 'symbol', 'tag'] else: if not isinstance(columns, list): raise ValueError( "ERR#0020: specified columns argument is not a list, it can just be list type." ) if not isinstance(as_json, bool): raise ValueError( "ERR#0002: as_json argument can just be True or False, bool type.") resource_package = __name__ resource_path = '/'.join(('resources', 'etfs', 'etfs.csv')) if pkg_resources.resource_exists(resource_package, resource_path): etfs = pd.read_csv( pkg_resources.resource_filename(resource_package, resource_path)) else: etfs = retrieve_etfs() if etfs is None: raise IOError("ERR#0009: etf list not found or unable to retrieve.") if not all(column in etfs.columns.tolist() for column in columns): raise ValueError( "ERR#0021: specified columns does not exist, available columns are <id, name, symbol, tag>" ) if country is None: if as_json: return json.dumps(etfs[columns].to_dict(orient='records')) else: return etfs[columns].to_dict(orient='records') elif country in retrieve_etf_countries(): if as_json: return json.dumps( etfs[etfs['country'] == country][columns].to_dict( orient='records')) else: return etfs[etfs['country'] == country][columns].to_dict( orient='records') # --- Useless Functions --- # def get_etf(country): # """ # This function retrieves all the available etfs to retrieve data from. # All the available etfs available can be found at: https://es.investing.com/etfs/spain-etfs # # Returns # ------- # :returns a dictionary containing all the etfs information # """ # # if country is None or not isinstance(country, str): # raise IOError("ERR#0028: specified country value not valid.") # # head = { # "User-Agent": ua.get_random(), # "X-Requested-With": "XMLHttpRequest", # "Accept": "text/html", # "Accept-Encoding": "gzip, deflate, br", # "Connection": "keep-alive", # } # # resource_package = __name__ # resource_path = '/'.join(('resources', 'etfs', 'etf_markets.csv')) # if pkg_resources.resource_exists(resource_package, resource_path): # markets = pd.read_csv(pkg_resources.resource_filename(resource_package, resource_path)) # else: # raise FileNotFoundError("ERR#0027: etf_markets file not found") # # for index, row in markets.iterrows(): # if row['country'] == country.lower(): # url = "https://es.investing.com/etfs/" + row['country'].replace(" ", "-") + "-etfs" # # req = requests.get(url, headers=head, timeout=15) # # if req.status_code != 200: # raise ConnectionError("ERR#0015: error " + str(req.status_code) + ", try again later.") # # root_ = fromstring(req.text) # path_ = root_.xpath(".//table[@id='etfs']" # "/tbody" # "/tr") # # results = list() # # if path_: # for elements_ in path_: # id_ = elements_.get('id').replace('pair_', '') # symbol = elements_.xpath(".//td[contains(@class, 'symbol')]")[0].get('title') # # nested = elements_.xpath(".//a")[0] # info = nested.get('href').replace('/etfs/', '') # # data = { # "name": nested.text, # "symbol": symbol, # "tag": info, # "id": id_ # } # # results.append(data) # else: # raise IOError("ERR#0026: specified country etfs not found or unable to retrieve.") # # df = pd.DataFrame(results) # # return df
def _module_exists(module_name): """Returns whether a nrunner "runner" module exists.""" module_filename = f'{module_name}.py' mod_path = os.path.join('plugins', 'runners', module_filename) return pkg_resources.resource_exists('avocado', mod_path)
def get_data_file(*dirs): path = os.path.join("data", *dirs) if not pkg_resources.resource_exists(__package__, path): raise FileNotFoundError("Data file '{}' not found".format(path)) return pkg_resources.resource_filename(__package__, path)
def main(argv=None): if argv is None: argv = sys.argv usage = "usage: %prog [options] [workflow_file]" parser = optparse.OptionParser(usage=usage) parser.add_option("--no-discovery", action="store_true", help="Don't run widget discovery " "(use full cache instead)") parser.add_option("--force-discovery", action="store_true", help="Force full widget discovery " "(invalidate cache)") parser.add_option("--no-welcome", action="store_true", help="Don't show welcome dialog.") parser.add_option("--no-splash", action="store_true", help="Don't show splash screen.") parser.add_option("-l", "--log-level", help="Logging level (0, 1, 2, 3, 4)", type="int", default=1) parser.add_option("--style", help="QStyle to use", type="str", default=None) parser.add_option("--stylesheet", help="Application level CSS style sheet to use", type="str", default=None) parser.add_option("--qt", help="Additional arguments for QApplication", type="str", default=None) parser.add_option("--config", help="Configuration namespace", type="str", default="orangecanvas.example") # -m canvas orange.widgets # -m canvas --config orange.widgets (options, args) = parser.parse_args(argv[1:]) levels = [logging.CRITICAL, logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG] # Fix streams before configuring logging (otherwise it will store # and write to the old file descriptors) fix_win_pythonw_std_stream() # Set http_proxy environment variable(s) for some clients fix_set_proxy_env() # Try to fix macOS automatic window tabbing (Sierra and later) fix_macos_nswindow_tabbing() # File handler should always be at least INFO level so we need # the application root level to be at least at INFO. root_level = min(levels[options.log_level], logging.INFO) rootlogger = logging.getLogger(__package__) rootlogger.setLevel(root_level) # Standard output stream handler at the requested level stream_hander = logging.StreamHandler() stream_hander.setLevel(level=levels[options.log_level]) rootlogger.addHandler(stream_hander) if options.config is not None: try: cfg = utils.name_lookup(options.config) except (ImportError, AttributeError): pass else: config.set_default(cfg()) log.info("activating %s", options.config) log.info("Starting 'Orange Canvas' application.") qt_argv = argv[:1] style = options.style defaultstylesheet = "orange.qss" fusiontheme = None if style is not None: if style.startswith("fusion:"): qt_argv += ["-style", "fusion"] _, _, fusiontheme = style.partition(":") else: qt_argv += ["-style", style] if options.qt is not None: qt_argv += shlex.split(options.qt) qt_argv += args if QT_VERSION >= 0x50600: CanvasApplication.setAttribute(Qt.AA_UseHighDpiPixmaps) log.debug("Starting CanvasApplicaiton with argv = %r.", qt_argv) app = CanvasApplication(qt_argv) if app.style().metaObject().className() == "QFusionStyle": if fusiontheme == "breeze-dark": app.setPalette(breeze_dark()) defaultstylesheet = "darkorange.qss" palette = app.palette() if style is None and palette.color(QPalette.Window).value() < 127: log.info("Switching default stylesheet to darkorange") defaultstylesheet = "darkorange.qss" # NOTE: config.init() must be called after the QApplication constructor config.init() file_handler = logging.FileHandler( filename=os.path.join(config.log_dir(), "canvas.log"), mode="w" ) file_handler.setLevel(root_level) rootlogger.addHandler(file_handler) # intercept any QFileOpenEvent requests until the main window is # fully initialized. # NOTE: The QApplication must have the executable ($0) and filename # arguments passed in argv otherwise the FileOpen events are # triggered for them (this is done by Cocoa, but QApplicaiton filters # them out if passed in argv) open_requests = [] def onrequest(url): log.info("Received an file open request %s", url) open_requests.append(url) app.fileOpenRequest.connect(onrequest) settings = QSettings() stylesheet = options.stylesheet or defaultstylesheet stylesheet_string = None if stylesheet != "none": if os.path.isfile(stylesheet): with io.open(stylesheet, "r") as f: stylesheet_string = f.read() else: if not os.path.splitext(stylesheet)[1]: # no extension stylesheet = os.path.extsep.join([stylesheet, "qss"]) pkg_name = __package__ resource = "styles/" + stylesheet if pkg_resources.resource_exists(pkg_name, resource): stylesheet_string = \ pkg_resources.resource_string(pkg_name, resource).decode("utf-8") base = pkg_resources.resource_filename(pkg_name, "styles") pattern = re.compile( r"^\s@([a-zA-Z0-9_]+?)\s*:\s*([a-zA-Z0-9_/]+?);\s*$", flags=re.MULTILINE ) matches = pattern.findall(stylesheet_string) for prefix, search_path in matches: QDir.addSearchPath(prefix, os.path.join(base, search_path)) log.info("Adding search path %r for prefix, %r", search_path, prefix) stylesheet_string = pattern.sub("", stylesheet_string) else: log.info("%r style sheet not found.", stylesheet) # Add the default canvas_icons search path dirpath = os.path.abspath(os.path.dirname(__file__)) QDir.addSearchPath("canvas_icons", os.path.join(dirpath, "icons")) canvas_window = CanvasMainWindow() canvas_window.setAttribute(Qt.WA_DeleteOnClose) canvas_window.setWindowIcon(config.application_icon()) if stylesheet_string is not None: canvas_window.setStyleSheet(stylesheet_string) if not options.force_discovery: reg_cache = cache.registry_cache() else: reg_cache = None widget_registry = qt.QtWidgetRegistry() widget_discovery = config.widget_discovery( widget_registry, cached_descriptions=reg_cache) want_splash = \ settings.value("startup/show-splash-screen", True, type=bool) and \ not options.no_splash if want_splash: pm, rect = config.splash_screen() splash_screen = SplashScreen(pixmap=pm, textRect=rect) splash_screen.setAttribute(Qt.WA_DeleteOnClose) splash_screen.setFont(QFont("Helvetica", 12)) color = QColor("#FFD39F") def show_message(message): splash_screen.showMessage(message, color=color) widget_registry.category_added.connect(show_message) show_splash = splash_screen.show close_splash = splash_screen.close else: show_splash = close_splash = lambda: None log.info("Running widget discovery process.") cache_filename = os.path.join(config.cache_dir(), "widget-registry.pck") if options.no_discovery: with open(cache_filename, "rb") as f: widget_registry = pickle.load(f) widget_registry = qt.QtWidgetRegistry(widget_registry) else: show_splash() widget_discovery.run(config.widgets_entry_points()) close_splash() # Store cached descriptions cache.save_registry_cache(widget_discovery.cached_descriptions) with open(cache_filename, "wb") as f: pickle.dump(WidgetRegistry(widget_registry), f) set_global_registry(widget_registry) canvas_window.set_widget_registry(widget_registry) canvas_window.show() canvas_window.raise_() want_welcome = \ settings.value("startup/show-welcome-screen", True, type=bool) \ and not options.no_welcome # Process events to make sure the canvas_window layout has # a chance to activate (the welcome dialog is modal and will # block the event queue, plus we need a chance to receive open file # signals when running without a splash screen) app.processEvents() app.fileOpenRequest.connect(canvas_window.open_scheme_file) if want_welcome and not args and not open_requests: canvas_window.welcome_dialog() elif args: log.info("Loading a scheme from the command line argument %r", args[0]) canvas_window.load_scheme(args[0]) elif open_requests: log.info("Loading a scheme from an `QFileOpenEvent` for %r", open_requests[-1]) canvas_window.load_scheme(open_requests[-1].toLocalFile()) # Tee stdout and stderr into Output dock output_view = canvas_window.output_view() stdout = TextStream() stdout.stream.connect(output_view.write) if sys.stdout: stdout.stream.connect(sys.stdout.write) stdout.flushed.connect(sys.stdout.flush) stderr = TextStream() error_writer = output_view.formated(color=Qt.red) stderr.stream.connect(error_writer.write) if sys.stderr: stderr.stream.connect(sys.stderr.write) stderr.flushed.connect(sys.stderr.flush) with ExitStack() as stack: stack.enter_context(closing(stderr)) stack.enter_context(closing(stdout)) stack.enter_context(redirect_stdout(stdout)) stack.enter_context(redirect_stderr(stderr)) log.info("Entering main event loop.") sys.excepthook = ExceptHook(stream=stderr) try: status = app.exec_() finally: sys.excepthook = sys.__excepthook__ del canvas_window app.processEvents() app.flush() # Collect any cycles before deleting the QApplication instance gc.collect() del app return status
def main(args, pass_through_args): conf = ClusterConfig() conf.extend_flag('image-version', IMAGE_VERSION) if not pkg_resources.resource_exists('hailtop.hailctl', "deploy.yaml"): raise RuntimeError(f"package has no 'deploy.yaml' file") deploy_metadata = yaml.safe_load( pkg_resources.resource_stream('hailtop.hailctl', "deploy.yaml"))['dataproc'] conf.extend_flag('properties', DEFAULT_PROPERTIES) if args.properties: conf.parse_and_extend('properties', args.properties) # default to highmem machines if using VEP if not args.worker_machine_type: args.worker_machine_type = 'n1-highmem-8' if args.vep else 'n1-standard-8' # default initialization script to start up cluster with conf.extend_flag('initialization-actions', [deploy_metadata['init_notebook.py']]) # add VEP init script if args.vep: conf.extend_flag('initialization-actions', [deploy_metadata[f'vep-{args.vep}.sh']]) # add custom init scripts if args.init: conf.extend_flag('initialization-actions', args.init.split(',')) if args.metadata: conf.parse_and_extend('metadata', args.metadata) wheel = args.wheel or deploy_metadata['wheel'] conf.extend_flag('metadata', {'WHEEL': wheel}) # if Python packages requested, add metadata variable packages = deploy_metadata['pip_dependencies'].strip('|').split('|||') metadata_pkgs = conf.flags['metadata'].get('PKGS') split_regex = r'[|,]' if metadata_pkgs: packages.extend(re.split(split_regex, metadata_pkgs)) if args.packages: packages.extend(re.split(split_regex, args.packages)) conf.extend_flag('metadata', {'PKGS': '|'.join(packages)}) def disk_size(size): if args.vep: size = max(size, 200) return str(size) + 'GB' # rewrite metadata to escape it conf.flags['metadata'] = '^|||^' + '|||'.join( f'{k}={v}' for k, v in conf.flags['metadata'].items()) conf.extend_flag( 'properties', { "spark:spark.driver.memory": "{driver_memory}g".format(driver_memory=str( int(MACHINE_MEM[args.master_machine_type] * args.master_memory_fraction))) }) conf.flags['master-machine-type'] = args.master_machine_type conf.flags['master-boot-disk-size'] = '{}GB'.format( args.master_boot_disk_size) conf.flags['num-master-local-ssds'] = args.num_master_local_ssds conf.flags['num-preemptible-workers'] = args.num_preemptible_workers conf.flags['num-worker-local-ssds'] = args.num_worker_local_ssds conf.flags['num-workers'] = args.num_workers conf.flags['preemptible-worker-boot-disk-size'] = disk_size( args.preemptible_worker_boot_disk_size) conf.flags['worker-boot-disk-size'] = disk_size(args.worker_boot_disk_size) conf.flags['worker-machine-type'] = args.worker_machine_type conf.flags['zone'] = args.zone conf.flags['initialization-action-timeout'] = args.init_timeout if args.network: conf.flags['network'] = args.network if args.configuration: conf.flags['configuration'] = args.configuration if args.project: conf.flags['project'] = args.project if args.bucket: conf.flags['bucket'] = args.bucket try: label = sp.check_output(['gcloud', 'config', 'get-value', 'account']) conf.flags['labels'] = 'creator=' + re.sub( r'[^0-9a-z_\-]', '_', label.decode().strip().lower())[:63] except sp.CalledProcessError as e: sys.stderr.write( "Warning: could not run 'gcloud config get-value account': " + e.output.decode() + "\n") # command to start cluster cmd = conf.get_command(args.name) if args.beta or args.max_idle or args.max_age: cmd.insert(1, 'beta') if args.max_idle: cmd.append('--max-idle={}'.format(args.max_idle)) if args.max_age: cmd.append('--max-age={}'.format(args.max_age)) cmd.extend(pass_through_args) # print underlying gcloud command print(' '.join(cmd[:5]) + ' \\\n ' + ' \\\n '.join(cmd[5:])) # spin up cluster if not args.dry_run: print("Starting cluster '{}'...".format(args.name)) sp.check_call(cmd) if args.master_tags: sp.check_call([ 'gcloud', 'compute', 'instances', 'add-tags', args.name + '-m', '--tags', args.master_tags ])
def main(args, pass_through_args): modify_args = [] if args.num_workers is not None: modify_args.append('--num-workers={}'.format(args.num_workers)) if args.num_preemptible_workers is not None: modify_args.append('--num-preemptible-workers={}'.format(args.num_preemptible_workers)) if args.graceful_decommission_timeout: if not modify_args: sys.exit("Error: Cannot use --graceful-decommission-timeout without resizing the cluster.") modify_args.append('--graceful-decommission-timeout={}'.format(args.graceful_decommission_timeout)) if args.max_idle: modify_args.append('--max-idle={}'.format(args.max_idle)) if args.update_hail_version and args.wheel: sys.exit("Error: Cannot specify both --update-hail-version and --wheel") if modify_args: cmd = ['gcloud', 'dataproc', 'clusters', 'update', args.name] + modify_args if args.beta: cmd.insert(1, 'beta') cmd.extend(pass_through_args) # print underlying gcloud command print(' '.join(cmd[:5]) + ' \\\n ' + ' \\\n '.join(cmd[5:])) # Update cluster if not args.dry_run: print("Updating cluster '{}'...".format(args.name)) check_call(cmd) wheel = None if args.update_hail_version: assert pkg_resources.resource_exists('hailtop.hailctl', "deploy.yaml") updated_wheel = yaml.safe_load( pkg_resources.resource_stream('hailtop.hailctl', "deploy.yaml"))['dataproc']['wheel'] wheel = updated_wheel else: wheel = args.wheel if wheel is not None: wheelfile = os.path.basename(wheel) cmds = [] if wheel.startswith("gs://"): cmds.append([ 'gcloud', 'compute', 'ssh', '{}-m'.format(args.name), '--zone={}'.format(args.zone), '--', f'sudo gsutil cp {wheel} /tmp/ && ' 'sudo /opt/conda/default/bin/pip uninstall -y hail && ' f'sudo /opt/conda/default/bin/pip install --no-dependencies /tmp/{wheelfile}' ]) else: cmds.extend([ [ 'gcloud', 'compute', 'scp', '--zone={}'.format(args.zone), wheel, '{}-m:/tmp/'.format(args.name) ], [ 'gcloud', 'compute', 'ssh', f'{args.name}-m', f'--zone={args.zone}', '--', 'sudo /opt/conda/default/bin/pip uninstall -y hail && ' f'sudo /opt/conda/default/bin/pip install --no-dependencies /tmp/{wheelfile}' ] ]) for cmd in cmds: print(cmd) check_call(cmd) if not wheel and not modify_args and pass_through_args: sys.stderr.write('ERROR: found pass-through arguments but not known modification args.') sys.exit(1)
def get_bond_information(bond, as_json=False): """ This function retrieves fundamental financial information from the specified bond. The retrieved information from the bond can be valuable as it is additional information that can be used combined with OHLC values, so to determine financial insights from the company which holds the specified bond. Args: bond (:obj:`str`): name of the bond to retrieve information from. as_json (:obj:`bool`, optional): optional argument to determine the format of the output data (:obj:`dict` or :obj:`json`). Returns: :obj:`pandas.DataFrame` or :obj:`dict`- bond_information: The resulting :obj:`pandas.DataFrame` contains the information fields retrieved from Investing.com from the specified bond; it can also be returned as a :obj:`dict`, if argument `as_json=True`. If any of the information fields could not be retrieved, that field/s will be filled with None values. If the retrieval process succeeded, the resulting :obj:`dict` will look like:: bond_information = { "1-Year Change": "46.91%", "52 wk Range": "-0.575 - 0.01", "Bond Name": "Spain 1Y", "Coupon": "None", "Maturity Date": "04/12/2020", "Prev. Close": -0.425, "Price": 100.417, "Price Open": 100.416, "Price Range": -100.481, "Todays Range": "-0.49 - -0.424" } Raises: ValueError: raised if any of the introduced arguments is not valid or errored. FileNotFoundError: raised if bonds.csv file was not found or errored. IOError: raised if bonds.csv file is empty or errored. RuntimeError: raised if scraping process failed while running. ConnectionError: raised if the connection to Investing.com errored (did not return HTTP 200) """ if not bond: raise ValueError( "ERR#0066: bond parameter is mandatory and must be a valid bond name." ) if not isinstance(bond, str): raise ValueError("ERR#0067: bond argument needs to be a str.") if not isinstance(as_json, bool): raise ValueError( "ERR#0002: as_json argument can just be True or False, bool type.") resource_package = 'investpy' resource_path = '/'.join(('resources', 'bonds.csv')) if pkg_resources.resource_exists(resource_package, resource_path): bonds = pd.read_csv( pkg_resources.resource_filename(resource_package, resource_path)) else: raise FileNotFoundError("ERR#0064: bonds file not found or errored.") if bonds is None: raise IOError( "ERR#0065: bonds object not found or unable to retrieve.") bond = unidecode(bond.strip().lower()) if bond not in list(bonds['name'].str.lower()): raise RuntimeError("ERR#0068: bond " + bond + " not found, check if it is correct.") name = bonds.loc[(bonds['name'].str.lower() == bond).idxmax(), 'name'] tag = bonds.loc[(bonds['name'].str.lower() == bond).idxmax(), 'tag'] url = "https://www.investing.com/rates-bonds/" + tag head = { "User-Agent": random_user_agent(), "X-Requested-With": "XMLHttpRequest", "Accept": "text/html", "Accept-Encoding": "gzip, deflate, br", "Connection": "keep-alive", } req = requests.get(url, headers=head) if req.status_code != 200: raise ConnectionError("ERR#0015: error " + str(req.status_code) + ", try again later.") root_ = fromstring(req.text) path_ = root_.xpath("//div[contains(@class, 'overviewDataTable')]/div") result = pd.DataFrame(columns=[ 'Bond Name', 'Prev. Close', 'Price', 'Coupon', 'Todays Range', 'Price Open', 'Maturity Date', '52 wk Range', 'Price Range', '1-Year Change' ]) result.at[0, 'Bond Name'] = name if path_: for elements_ in path_: element = elements_.xpath(".//span[@class='float_lang_base_1']")[0] title_ = element.text_content() if title_ == "Day's Range": title_ = 'Todays Range' if title_ in result.columns.tolist(): try: result.at[0, title_] = float( element.getnext().text_content().replace(',', '')) continue except: pass try: text = element.getnext().text_content().strip() result.at[0, title_] = datetime.strptime( text, "%d %b %Y").strftime("%d/%m/%Y") continue except: pass try: text = element.getnext().text_content().strip() occ = text.count('-') if occ == 1: reg = re.compile(r"([\-]{1}[ ]{1}[0-9\.]+)+") matches = reg.findall(text) if len(matches) > 0: result.at[0, title_] = float(matches[0].replace( ' ', '')) continue elif occ == 2: reg = re.compile(r"([\-]{1}[ ]{1}[0-9\.]+)+") matches = reg.findall(text) if len(matches) > 0: res = matches[0].replace(' ', '') result.at[0, title_] = ' '.join([res, matches[1]]) continue elif occ == 3: reg = re.compile(r"([\-]{1}[ ]{1}[0-9\.]+)+") matches = reg.findall(text) if len(matches) > 0: res = list() for match in matches: res.append(match.replace(' ', '')) result.at[0, title_] = ' - '.join(res) continue except: pass try: value = element.getnext().text_content().strip() if value.__contains__('K'): value = float(value.replace('K', '').replace(',', '')) * 1e3 elif value.__contains__('M'): value = float(value.replace('M', '').replace(',', '')) * 1e6 elif value.__contains__('B'): value = float(value.replace('B', '').replace(',', '')) * 1e9 elif value.__contains__('T'): value = float(value.replace('T', '').replace( ',', '')) * 1e12 result.at[0, title_] = value continue except: pass result.replace({'N/A': None}, inplace=True) if as_json is True: json_ = result.iloc[0].to_dict() return json_ elif as_json is False: return result else: raise RuntimeError("ERR#0004: data retrieval error while scraping.")
def get_currency_cross_historical_data(currency_cross, from_date, to_date, as_json=False, order='ascending', interval='Daily'): """ This function retrieves recent historical data from the introduced `currency_cross` from Investing via Web Scraping. The resulting data can it either be stored in a :obj:`pandas.DataFrame` or in a :obj:`json` file, with `ascending` or `descending` order. Args: currency_cross (:obj:`str`): name of the currency cross to retrieve recent historical data from. from_date (:obj:`str`): date as `str` formatted as `dd/mm/yyyy`, from where data is going to be retrieved. to_date (:obj:`str`): date as `str` formatted as `dd/mm/yyyy`, until where data is going to be retrieved. as_json (:obj:`bool`, optional): optional argument to determine the format of the output data (:obj:`pandas.DataFrame` or :obj:`json`). order (:obj:`str`, optional): optional argument to define the order of the retrieved data (`ascending`, `asc` or `descending`, `desc`). interval (:obj:`str`, optional): value to define the historical data interval to retrieve, by default `Daily`, but it can also be `Weekly` or `Monthly`. Returns: :obj:`pandas.DataFrame` or :obj:`json`: The function returns a either a :obj:`pandas.DataFrame` or a :obj:`json` file containing the retrieved recent data from the specified currency_cross via argument. The dataset contains the open, high, low, close and volume values for the selected currency_cross on market days. The return data is case we use default arguments will look like:: Date || Open | High | Low | Close | Currency -----||------|------|-----|-------|--------- xxxx || xxxx | xxxx | xxx | xxxxx | xxxxxxxx but if we define `as_json=True`, then the output will be:: { name: name, historical: [ dd/mm/yyyy: { 'open': x, 'high': x, 'low': x, 'close': x, 'currency' : x }, ... ] } Raises: ValueError: argument error. IOError: stocks object/file not found or unable to retrieve. RuntimeError: introduced currency_cross does not match any of the indexed ones. ConnectionError: if GET requests does not return 200 status code. IndexError: if currency_cross information was unavailable or not found. Examples: >>> data = investpy.get_currency_cross_historical_data(currency_cross='EUR/USD', from_date='01/01/2018', to_date='01/01/2019') >>> data.head() Open High Low Close Currency Date 2018-01-01 1.2003 1.2014 1.1995 1.2010 USD 2018-01-02 1.2013 1.2084 1.2003 1.2059 USD 2018-01-03 1.2058 1.2070 1.2001 1.2014 USD 2018-01-04 1.2015 1.2090 1.2004 1.2068 USD 2018-01-05 1.2068 1.2085 1.2021 1.2030 USD """ if not currency_cross: raise ValueError("ERR#0052: currency_cross param is mandatory and should be a str.") if not isinstance(currency_cross, str): raise ValueError("ERR#0052: currency_cross param is mandatory and should be a str.") try: datetime.strptime(from_date, '%d/%m/%Y') except ValueError: raise ValueError("ERR#0011: incorrect data format, it should be 'dd/mm/yyyy'.") try: datetime.strptime(to_date, '%d/%m/%Y') except ValueError: raise ValueError("ERR#0011: incorrect data format, it should be 'dd/mm/yyyy'.") start_date = datetime.strptime(from_date, '%d/%m/%Y') end_date = datetime.strptime(to_date, '%d/%m/%Y') if start_date >= end_date: raise ValueError("ERR#0032: to_date should be greater than from_date, both formatted as 'dd/mm/yyyy'.") if not isinstance(as_json, bool): raise ValueError("ERR#0002: as_json argument can just be True or False, bool type.") if order not in ['ascending', 'asc', 'descending', 'desc']: raise ValueError("ERR#0003: order argument can just be ascending (asc) or descending (desc), str type.") if not interval: raise ValueError("ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'.") if not isinstance(interval, str): raise ValueError("ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'.") if interval not in ['Daily', 'Weekly', 'Monthly']: raise ValueError("ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'.") date_interval = { 'intervals': [], } flag = True while flag is True: diff = end_date.year - start_date.year if diff > 19: obj = { 'start': start_date.strftime('%m/%d/%Y'), 'end': start_date.replace(year=start_date.year + 19).strftime('%m/%d/%Y'), } date_interval['intervals'].append(obj) start_date = start_date.replace(year=start_date.year + 19) + timedelta(days=1) else: obj = { 'start': start_date.strftime('%m/%d/%Y'), 'end': end_date.strftime('%m/%d/%Y'), } date_interval['intervals'].append(obj) flag = False interval_limit = len(date_interval['intervals']) interval_counter = 0 data_flag = False resource_package = 'investpy' resource_path = '/'.join(('resources', 'currency_crosses.csv')) if pkg_resources.resource_exists(resource_package, resource_path): currency_crosses = pd.read_csv(pkg_resources.resource_filename(resource_package, resource_path)) else: raise FileNotFoundError("ERR#0060: currency_crosses file not found or errored.") if currency_crosses is None: raise IOError("ERR#0050: currency_crosses not found or unable to retrieve.") currency_cross = currency_cross.strip() currency_cross = currency_cross.lower() if unidecode(currency_cross) not in [unidecode(value.lower()) for value in currency_crosses['name'].tolist()]: raise RuntimeError("ERR#0054: the introduced currency_cross " + str(currency_cross) + " does not exists.") id_ = currency_crosses.loc[(currency_crosses['name'].str.lower() == currency_cross).idxmax(), 'id'] name = currency_crosses.loc[(currency_crosses['name'].str.lower() == currency_cross).idxmax(), 'name'] currency = currency_crosses.loc[(currency_crosses['name'].str.lower() == currency_cross).idxmax(), 'second'] final = list() header = name + ' Historical Data' for index in range(len(date_interval['intervals'])): interval_counter += 1 params = { "curr_id": id_, "smlID": str(randint(1000000, 99999999)), "header": header, "st_date": date_interval['intervals'][index]['start'], "end_date": date_interval['intervals'][index]['end'], "interval_sec": interval, "sort_col": "date", "sort_ord": "DESC", "action": "historical_data" } head = { "User-Agent": random_user_agent(), "X-Requested-With": "XMLHttpRequest", "Accept": "text/html", "Accept-Encoding": "gzip, deflate, br", "Connection": "keep-alive", } url = "https://www.investing.com/instruments/HistoricalDataAjax" req = requests.post(url, headers=head, data=params) if req.status_code != 200: raise ConnectionError("ERR#0015: error " + str(req.status_code) + ", try again later.") if not req.text: continue root_ = fromstring(req.text) path_ = root_.xpath(".//table[@id='curr_table']/tbody/tr") result = list() if path_: for elements_ in path_: info = [] for nested_ in elements_.xpath(".//td"): info.append(nested_.get('data-real-value')) if elements_.xpath(".//td")[0].text_content() == 'No results found': if interval_counter < interval_limit: data_flag = False else: raise IndexError("ERR#0055: currency_cross information unavailable or not found.") else: data_flag = True if data_flag is True: currency_cross_date = datetime.strptime(str(datetime.fromtimestamp(int(info[0]), tz=pytz.utc).date()), '%Y-%m-%d') currency_cross_close = float(info[1].replace(',', '')) currency_cross_open = float(info[2].replace(',', '')) currency_cross_high = float(info[3].replace(',', '')) currency_cross_low = float(info[4].replace(',', '')) result.insert(len(result), Data(currency_cross_date, currency_cross_open, currency_cross_high, currency_cross_low, currency_cross_close, None, currency, None)) if data_flag is True: if order in ['ascending', 'asc']: result = result[::-1] elif order in ['descending', 'desc']: result = result if as_json is True: json_ = { 'name': name, 'historical': [value.currency_cross_as_json() for value in result] } final.append(json_) elif as_json is False: df = pd.DataFrame.from_records([value.currency_cross_to_dict() for value in result]) df.set_index('Date', inplace=True) final.append(df) else: raise RuntimeError("ERR#0004: data retrieval error while scraping.") if as_json is True: return json.dumps(final[0], sort_keys=False) elif as_json is False: return pd.concat(final)
def load_manifest_schema(): resource_args = (__name__, '../manifest_schema.yaml') assert pkg_resources.resource_exists(*resource_args) yaml_data = pkg_resources.resource_string(*resource_args) return ruamel.yaml.load(yaml_data)
def get_bonds_overview(country, as_json=False): """ This function retrieves an overview containing all the real time data available for the government bonds from a country, such as the names, symbols, current value, etc. as indexed in Investing.com. So on, the main usage of this function is to get an overview on the government bonds from a country, so to get a general view. Args: country (:obj:`str`): name of the country to retrieve the government bonds overview from. as_json (:obj:`bool`, optional): optional argument to determine the format of the output data (:obj:`pandas.DataFrame` or :obj:`json`). Returns: :obj:`pandas.DataFrame` - bonds_overview: The resulting :obj:`pandas.DataFrame` contains all the data available in Investing.com of the government bonds from a country in order to get an overview of it. If the retrieval process succeeded, the resulting :obj:`pandas.DataFrame` should look like:: country | name | last | last_close | high | low | change | change_percentage --------|------|------|------------|------|-----|--------|------------------- xxxxxxx | xxxx | xxxx | xxxxxxxxxx | xxxx | xxx | xxxxxx | xxxxxxxxxxxxxxxxx Raises: ValueError: raised if any of the introduced arguments is not valid or errored. FileNotFoundError: raised if `bonds.csv` file is missing. IOError: raised if data could not be retrieved due to file error. RuntimeError: raised either if the introduced country does not match any of the listed ones or if no overview results could be retrieved from Investing.com. ConnectionError: raised if GET requests does not return 200 status code. """ if country is None: raise ValueError( "ERR#0039: country can not be None, it should be a str.") if country is not None and not isinstance(country, str): raise ValueError("ERR#0025: specified country value not valid.") if not isinstance(as_json, bool): raise ValueError( "ERR#0002: as_json argument can just be True or False, bool type.") resource_package = 'investpy' resource_path = '/'.join(('resources', 'bonds.csv')) if pkg_resources.resource_exists(resource_package, resource_path): bonds = pd.read_csv( pkg_resources.resource_filename(resource_package, resource_path)) else: raise FileNotFoundError("ERR#0064: bonds file not found or errored.") if bonds is None: raise IOError( "ERR#0065: bonds object not found or unable to retrieve.") country = unidecode(country.strip().lower()) if country not in get_bond_countries(): raise ValueError("ERR#0034: country " + country + " not found, check if it is correct.") bonds = bonds[bonds['country'] == country] if country == 'united states': country = 'usa' elif country == 'united kingdom': country = 'uk' head = { "User-Agent": random_user_agent(), "X-Requested-With": "XMLHttpRequest", "Accept": "text/html", "Accept-Encoding": "gzip, deflate, br", "Connection": "keep-alive", } url = 'https://www.investing.com/rates-bonds/' + country + '-government-bonds' req = requests.get(url, headers=head) if req.status_code != 200: raise ConnectionError("ERR#0015: error " + str(req.status_code) + ", try again later.") root_ = fromstring(req.text) table = root_.xpath(".//table[@id='cr1']/tbody/tr") results = list() if len(table) > 0: for row in table: id_ = row.get('id').replace('pair_', '') country_check = row.xpath(".//td[@class='flag']/span")[0].get( 'title').lower() name = row.xpath( ".//td[contains(@class, 'elp')]/a")[0].text_content().strip() pid = 'pid-' + id_ last = row.xpath(".//td[@class='" + pid + "-last']")[0].text_content() last_close = row.xpath(".//td[@class='" + pid + "-last_close']")[0].text_content() high = row.xpath(".//td[@class='" + pid + "-high']")[0].text_content() low = row.xpath(".//td[@class='" + pid + "-low']")[0].text_content() pc = row.xpath(".//td[contains(@class, '" + pid + "-pc')]")[0].text_content() pcp = row.xpath(".//td[contains(@class, '" + pid + "-pcp')]")[0].text_content() data = { "country": country_check, "name": name, "last": float(last.replace(',', '')), "last_close": float(last_close.replace(',', '')), "high": float(high.replace(',', '')), "low": float(low.replace(',', '')), "change": pc, "change_percentage": pcp } results.append(data) else: raise RuntimeError( "ERR#0092: no data found while retrieving the overview from Investing.com" ) df = pd.DataFrame(results) if as_json: return json.loads(df.to_json(orient='records')) else: return df
def get_currency_cross_recent_data(currency_cross, as_json=False, order='ascending', interval='Daily'): """ This function retrieves recent historical data from the introduced `currency_cross` as indexed in Investing.com via Web Scraping. The resulting data can it either be stored in a :obj:`pandas.DataFrame` or in a :obj:`json` file, with `ascending` or `descending` order. Args: currency_cross (:obj:`str`): name of the currency_cross to retrieve recent historical data from. as_json (:obj:`bool`, optional): optional argument to determine the format of the output data (:obj:`pandas.DataFrame` or :obj:`json`). order (:obj:`str`, optional): optional argument to define the order of the retrieved data (`ascending`, `asc` or `descending`, `desc`). interval (:obj:`str`, optional): value to define the historical data interval to retrieve, by default `Daily`, but it can also be `Weekly` or `Monthly`. Returns: :obj:`pandas.DataFrame` or :obj:`json`: The function returns a either a :obj:`pandas.DataFrame` or a :obj:`json` file containing the retrieved recent data from the specified currency_cross via argument. The dataset contains the open, high, low, close, volume and currency values for the selected currency_cross on market days. The return data is in case we use default arguments will look like:: Date || Open | High | Low | Close | Currency -----||------|------|-----|-------|--------- xxxx || xxxx | xxxx | xxx | xxxxx | xxxxxxxx but if we define `as_json=True`, then the output will be:: { name: name, recent: [ dd/mm/yyyy: { 'open': x, 'high': x, 'low': x, 'close': x, 'currency' : x }, ... ] } Raises: ValueError: raised if any of the introduced arguments was not valid or errored. IOError: raised if currency_crosses object/file not found or unable to retrieve. RuntimeError: raised introduced currency_cross does not match any of the indexed ones. ConnectionError: raised if GET request did not return 200 status code. IndexError: raised if currency_cross information was unavailable or not found. Examples: >>> data = investpy.get_currency_cross_recent_data(currency_cross='EUR/USD') >>> data.head() Open High Low Close Currency Date 2019-08-27 1.1101 1.1116 1.1084 1.1091 USD 2019-08-28 1.1090 1.1099 1.1072 1.1078 USD 2019-08-29 1.1078 1.1093 1.1042 1.1057 USD 2019-08-30 1.1058 1.1062 1.0963 1.0991 USD 2019-09-02 1.0990 1.1000 1.0958 1.0968 USD """ if not currency_cross: raise ValueError("ERR#0052: currency_cross param is mandatory and should be a str.") if not isinstance(currency_cross, str): raise ValueError("ERR#0052: currency_cross param is mandatory and should be a str.") if not isinstance(as_json, bool): raise ValueError("ERR#0002: as_json argument can just be True or False, bool type.") if order not in ['ascending', 'asc', 'descending', 'desc']: raise ValueError("ERR#0003: order argument can just be ascending (asc) or descending (desc), str type.") if not interval: raise ValueError("ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'.") if not isinstance(interval, str): raise ValueError("ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'.") if interval not in ['Daily', 'Weekly', 'Monthly']: raise ValueError("ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'.") resource_package = 'investpy' resource_path = '/'.join(('resources', 'currency_crosses.csv')) if pkg_resources.resource_exists(resource_package, resource_path): currency_crosses = pd.read_csv(pkg_resources.resource_filename(resource_package, resource_path)) else: raise FileNotFoundError("ERR#0060: currency_crosses file not found or errored.") if currency_crosses is None: raise IOError("ERR#0050: currency_crosses not found or unable to retrieve.") currency_cross = currency_cross.strip() currency_cross = currency_cross.lower() if unidecode(currency_cross) not in [unidecode(value.lower()) for value in currency_crosses['name'].tolist()]: raise RuntimeError("ERR#0054: the introduced currency_cross " + str(currency_cross) + " does not exists.") id_ = currency_crosses.loc[(currency_crosses['name'].str.lower() == currency_cross).idxmax(), 'id'] name = currency_crosses.loc[(currency_crosses['name'].str.lower() == currency_cross).idxmax(), 'name'] currency = currency_crosses.loc[(currency_crosses['name'].str.lower() == currency_cross).idxmax(), 'second'] header = name + ' Historical Data' params = { "curr_id": id_, "smlID": str(randint(1000000, 99999999)), "header": header, "interval_sec": interval, "sort_col": "date", "sort_ord": "DESC", "action": "historical_data" } head = { "User-Agent": random_user_agent(), "X-Requested-With": "XMLHttpRequest", "Accept": "text/html", "Accept-Encoding": "gzip, deflate, br", "Connection": "keep-alive", } url = "https://www.investing.com/instruments/HistoricalDataAjax" req = requests.post(url, headers=head, data=params) if req.status_code != 200: raise ConnectionError("ERR#0015: error " + str(req.status_code) + ", try again later.") root_ = fromstring(req.text) path_ = root_.xpath(".//table[@id='curr_table']/tbody/tr") result = list() if path_: for elements_ in path_: if elements_.xpath(".//td")[0].text_content() == 'No results found': raise IndexError("ERR#0055: currency_cross information unavailable or not found.") info = [] for nested_ in elements_.xpath(".//td"): info.append(nested_.get('data-real-value')) currency_cross_date = datetime.strptime(str(datetime.fromtimestamp(int(info[0]), tz=pytz.utc).date()), '%Y-%m-%d') currency_cross_close = float(info[1].replace(',', '')) currency_cross_open = float(info[2].replace(',', '')) currency_cross_high = float(info[3].replace(',', '')) currency_cross_low = float(info[4].replace(',', '')) result.insert(len(result), Data(currency_cross_date, currency_cross_open, currency_cross_high, currency_cross_low, currency_cross_close, None, currency, None)) if order in ['ascending', 'asc']: result = result[::-1] elif order in ['descending', 'desc']: result = result if as_json is True: json_ = { 'name': name, 'recent': [value.currency_cross_as_json() for value in result] } return json.dumps(json_, sort_keys=False) elif as_json is False: df = pd.DataFrame.from_records([value.currency_cross_to_dict() for value in result]) df.set_index('Date', inplace=True) return df else: raise RuntimeError("ERR#0004: data retrieval error while scraping.")
def get_bond_historical_data(bond, from_date, to_date, as_json=False, order='ascending', interval='Daily'): """ This function retrieves historical data from the introduced bond from Investing.com. So on, the historical data of the introduced bond in the specified date range will be retrieved and returned as a :obj:`pandas.DataFrame` if the parameters are valid and the request to Investing.com succeeds. Note that additionally some optional parameters can be specified: as_json and order, which let the user decide if the data is going to be returned as a :obj:`json` or not, and if the historical data is going to be ordered ascending or descending (where the index is the date), respectively. Args: bond (:obj:`str`): name of the bond to retrieve historical data from. from_date (:obj:`str`): date formatted as `dd/mm/yyyy`, since when data is going to be retrieved. to_date (:obj:`str`): date formatted as `dd/mm/yyyy`, until when data is going to be retrieved. as_json (:obj:`bool`, optional): to determine the format of the output data, either a :obj:`pandas.DataFrame` if False and a :obj:`json` if True. order (:obj:`str`, optional): to define the order of the retrieved data which can either be ascending or descending. interval (:obj:`str`, optional): value to define the historical data interval to retrieve, by default `Daily`, but it can also be `Weekly` or `Monthly`. Returns: :obj:`pandas.DataFrame` or :obj:`json`: The function returns a either a :obj:`pandas.DataFrame` or a :obj:`json` file containing the retrieved recent data from the specified bond via argument. The dataset contains the open, high, low and close for the selected bond on market days. The resulting recent data, in case that the default parameters were applied, will look like:: Date || Open | High | Low | Close -----||------|------|-----|------- xxxx || xxxx | xxxx | xxx | xxxxx but in case that as_json parameter was defined as True, then the output will be:: { name: name, historical: [ { date: 'dd/mm/yyyy', open: x, high: x, low: x, close: x }, ... ] } Raises: ValueError: raised whenever any of the introduced arguments is not valid or errored. IOError: raised if bonds object/file was not found or unable to retrieve. RuntimeError: raised if the introduced bond was not found or did not match any of the existing ones. ConnectionError: raised if connection to Investing.com could not be established. IndexError: raised if bond historical data was unavailable or not found in Investing.com. Examples: >>> data = investpy.get_bond_historical_data(bond='Argentina 3Y', from_date='01/01/2010', to_date='01/01/2019') >>> data.head() Open High Low Close Date 2011-01-03 4.15 4.15 4.15 5.15 2011-01-04 4.07 4.07 4.07 5.45 2011-01-05 4.27 4.27 4.27 5.71 2011-01-10 4.74 4.74 4.74 6.27 2011-01-11 4.30 4.30 4.30 6.56 """ if not bond: raise ValueError( "ERR#0066: bond parameter is mandatory and must be a valid bond name." ) if not isinstance(bond, str): raise ValueError("ERR#0067: bond argument needs to be a str.") if not isinstance(as_json, bool): raise ValueError( "ERR#0002: as_json argument can just be True or False, bool type.") if order not in ['ascending', 'asc', 'descending', 'desc']: raise ValueError( "ERR#0003: order argument can just be ascending (asc) or descending (desc), str type." ) if not interval: raise ValueError( "ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'." ) if not isinstance(interval, str): raise ValueError( "ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'." ) interval = interval.lower() if interval not in ['daily', 'weekly', 'monthly']: raise ValueError( "ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'." ) try: datetime.strptime(from_date, '%d/%m/%Y') except ValueError: raise ValueError( "ERR#0011: incorrect from_date date format, it should be 'dd/mm/yyyy'." ) try: datetime.strptime(to_date, '%d/%m/%Y') except ValueError: raise ValueError( "ERR#0012: incorrect to_date format, it should be 'dd/mm/yyyy'.") start_date = datetime.strptime(from_date, '%d/%m/%Y') end_date = datetime.strptime(to_date, '%d/%m/%Y') if start_date >= end_date: raise ValueError( "ERR#0032: to_date should be greater than from_date, both formatted as 'dd/mm/yyyy'." ) date_interval = { 'intervals': [], } flag = True while flag is True: diff = end_date.year - start_date.year if diff > 19: obj = { 'start': start_date.strftime('%m/%d/%Y'), 'end': start_date.replace(year=start_date.year + 19).strftime('%m/%d/%Y'), } date_interval['intervals'].append(obj) start_date = start_date.replace(year=start_date.year + 19) + timedelta(days=1) else: obj = { 'start': start_date.strftime('%m/%d/%Y'), 'end': end_date.strftime('%m/%d/%Y'), } date_interval['intervals'].append(obj) flag = False interval_limit = len(date_interval['intervals']) interval_counter = 0 data_flag = False resource_package = 'investpy' resource_path = '/'.join(('resources', 'bonds.csv')) if pkg_resources.resource_exists(resource_package, resource_path): bonds = pd.read_csv( pkg_resources.resource_filename(resource_package, resource_path)) else: raise FileNotFoundError("ERR#0064: bonds file not found or errored.") if bonds is None: raise IOError( "ERR#0065: bonds object not found or unable to retrieve.") bond = unidecode(bond.strip().lower()) if bond not in list(bonds['name'].str.lower()): raise RuntimeError("ERR#0068: bond " + bond + " not found, check if it is correct.") id_ = bonds.loc[(bonds['name'].str.lower() == bond).idxmax(), 'id'] name = bonds.loc[(bonds['name'].str.lower() == bond).idxmax(), 'name'] full_name = bonds.loc[(bonds['name'].str.lower() == bond).idxmax(), 'full_name'] final = list() header = full_name + " Bond Yield Historical Data" for index in range(len(date_interval['intervals'])): interval_counter += 1 params = { "curr_id": id_, "smlID": str(randint(1000000, 99999999)), "header": header, "st_date": date_interval['intervals'][index]['start'], "end_date": date_interval['intervals'][index]['end'], "interval_sec": interval.capitalize(), "sort_col": "date", "sort_ord": "DESC", "action": "historical_data" } head = { "User-Agent": random_user_agent(), "X-Requested-With": "XMLHttpRequest", "Accept": "text/html", "Accept-Encoding": "gzip, deflate, br", "Connection": "keep-alive", } url = "https://www.investing.com/instruments/HistoricalDataAjax" req = requests.post(url, headers=head, data=params) if req.status_code != 200: raise ConnectionError("ERR#0015: error " + str(req.status_code) + ", try again later.") if not req.text: continue root_ = fromstring(req.text) path_ = root_.xpath(".//table[@id='curr_table']/tbody/tr") result = list() if path_: for elements_ in path_: if elements_.xpath( ".//td")[0].text_content() == 'No results found': if interval_counter < interval_limit: data_flag = False else: raise IndexError( "ERR#0069: bond information unavailable or not found." ) else: data_flag = True if data_flag is True: info = [] for nested_ in elements_.xpath(".//td"): info.append(nested_.get('data-real-value')) bond_date = datetime.strptime( str( datetime.fromtimestamp( int(info[0]), tz=pytz.timezone('GMT')).date()), '%Y-%m-%d') bond_close = float(info[1].replace(',', '')) bond_open = float(info[2].replace(',', '')) bond_high = float(info[3].replace(',', '')) bond_low = float(info[4].replace(',', '')) result.insert( len(result), Data(bond_date, bond_open, bond_high, bond_low, bond_close, None, None, None)) if data_flag is True: if order in ['ascending', 'asc']: result = result[::-1] elif order in ['descending', 'desc']: result = result if as_json is True: json_list = [value.bond_as_json() for value in result] final.append(json_list) elif as_json is False: df = pd.DataFrame.from_records( [value.bond_to_dict() for value in result]) df.set_index('Date', inplace=True) final.append(df) else: raise RuntimeError( "ERR#0004: data retrieval error while scraping.") if order in ['descending', 'desc']: final.reverse() if as_json is True: json_ = { 'name': name, 'historical': [value for json_list in final for value in json_list] } return json.dumps(json_, sort_keys=False) elif as_json is False: return pd.concat(final)
def __init__(self, sc=None, app_name="Hail", master=None, local='local[*]', log=None, quiet=False, append=False, min_block_size=1, branching_factor=50, tmp_dir=None, default_reference="GRCh37", idempotent=False, global_seed=6348563392232659379, _backend=None): if Env._hc: if idempotent: return else: raise FatalError( 'Hail has already been initialized, restart session ' 'or stop Hail to change configuration.') if pkg_resources.resource_exists(__name__, "hail-all-spark.jar"): hail_jar_path = pkg_resources.resource_filename( __name__, "hail-all-spark.jar") assert os.path.exists( hail_jar_path), f'{hail_jar_path} does not exist' sys.stderr.write(f'using hail jar at {hail_jar_path}\n') conf = SparkConf() conf.set('spark.driver.extraClassPath', hail_jar_path) conf.set('spark.executor.extraClassPath', hail_jar_path) SparkContext._ensure_initialized(conf=conf) else: SparkContext._ensure_initialized() self._gateway = SparkContext._gateway self._jvm = SparkContext._jvm # hail package self._hail = getattr(self._jvm, 'is').hail self._warn_cols_order = True self._warn_entries_order = True Env._jvm = self._jvm Env._gateway = self._gateway jsc = sc._jsc.sc() if sc else None if _backend is None: apiserver_url = os.environ.get('HAIL_APISERVER_URL') if apiserver_url is not None: _backend = ServiceBackend(apiserver_url) else: _backend = SparkBackend() self._backend = _backend tmp_dir = get_env_or_default(tmp_dir, 'TMPDIR', '/tmp') version = read_version_info() hail.__version__ = version if log is None: log = hail.utils.timestamp_path(os.path.join(os.getcwd(), 'hail'), suffix=f'-{version}.log') self._log = log # we always pass 'quiet' to the JVM because stderr output needs # to be routed through Python separately. # if idempotent: if idempotent: self._jhc = self._hail.HailContext.getOrCreate( jsc, app_name, joption(master), local, log, True, append, min_block_size, branching_factor, tmp_dir) else: self._jhc = self._hail.HailContext.apply(jsc, app_name, joption(master), local, log, True, append, min_block_size, branching_factor, tmp_dir) self._jsc = self._jhc.sc() self.sc = sc if sc else SparkContext( gateway=self._gateway, jsc=self._jvm.JavaSparkContext(self._jsc)) self._jsql_context = self._jhc.sqlContext() self._sql_context = SQLContext(self.sc, jsqlContext=self._jsql_context) super(HailContext, self).__init__() # do this at the end in case something errors, so we don't raise the above error without a real HC Env._hc = self ReferenceGenome._from_config(_backend.get_reference('GRCh37'), True) ReferenceGenome._from_config(_backend.get_reference('GRCh38'), True) ReferenceGenome._from_config(_backend.get_reference('GRCm38'), True) if default_reference in ReferenceGenome._references: self._default_ref = ReferenceGenome._references[default_reference] else: self._default_ref = ReferenceGenome.read(default_reference) jar_version = self._jhc.version() if jar_version != version: raise RuntimeError( f"Hail version mismatch between JAR and Python library\n" f" JAR: {jar_version}\n" f" Python: {version}") if not quiet: sys.stderr.write('Running on Apache Spark version {}\n'.format( self.sc.version)) if self._jsc.uiWebUrl().isDefined(): sys.stderr.write('SparkUI available at {}\n'.format( self._jsc.uiWebUrl().get())) connect_logger('localhost', 12888) self._hail.HailContext.startProgressBar(self._jsc) sys.stderr.write( 'Welcome to\n' ' __ __ <>__\n' ' / /_/ /__ __/ /\n' ' / __ / _ `/ / /\n' ' /_/ /_/\\_,_/_/_/ version {}\n'.format(version)) if version.startswith('devel'): sys.stderr.write( 'NOTE: This is a beta version. Interfaces may change\n' ' during the beta period. We recommend pulling\n' ' the latest changes weekly.\n') sys.stderr.write(f'LOGGING: writing to {log}\n') install_exception_handler() Env.set_seed(global_seed)
def has_gff3(locus, segment, collection=None): file_name = _get_file_name(locus, segment, collection=collection, extension='.gff3') return resource_exists(_PKG, file_name)
def main(args, pass_through_args): conf = ClusterConfig() conf.extend_flag('image-version', IMAGE_VERSION) if not pkg_resources.resource_exists('hailtop.hailctl', "deploy.yaml"): raise RuntimeError("package has no 'deploy.yaml' file") deploy_metadata = yaml.safe_load( pkg_resources.resource_stream('hailtop.hailctl', "deploy.yaml"))['dataproc'] conf.extend_flag('properties', DEFAULT_PROPERTIES) if args.properties: conf.parse_and_extend('properties', args.properties) if args.debug_mode: conf.extend_flag( 'properties', { "spark:spark.driver.extraJavaOptions": "-Xss4M -XX:+HeapDumpOnOutOfMemoryError", "spark:spark.executor.extraJavaOptions": "-Xss4M -XX:+HeapDumpOnOutOfMemoryError", }) # default to highmem machines if using VEP if not args.worker_machine_type: args.worker_machine_type = 'n1-highmem-8' if args.vep else 'n1-standard-8' # default initialization script to start up cluster with conf.extend_flag('initialization-actions', [deploy_metadata['init_notebook.py']]) # requester pays support if args.requester_pays_allow_all or args.requester_pays_allow_buckets or args.requester_pays_allow_annotation_db: if args.requester_pays_allow_all and args.requester_pays_allow_buckets: raise RuntimeError( "Cannot specify both 'requester_pays_allow_all' and 'requester_pays_allow_buckets" ) if args.requester_pays_allow_all: requester_pays_mode = "AUTO" else: requester_pays_mode = "CUSTOM" requester_pays_bucket_sources = [] if args.requester_pays_allow_buckets: requester_pays_bucket_sources.append( args.requester_pays_allow_buckets) if args.requester_pays_allow_annotation_db: requester_pays_bucket_sources.extend(ANNOTATION_DB_BUCKETS) conf.extend_flag( "properties", { "spark:spark.hadoop.fs.gs.requester.pays.buckets": ",".join(requester_pays_bucket_sources) }) # Need to pick requester pays project. requester_pays_project = args.project if args.project else gcloud.get_config( "project") conf.extend_flag( "properties", { "spark:spark.hadoop.fs.gs.requester.pays.mode": requester_pays_mode, "spark:spark.hadoop.fs.gs.requester.pays.project.id": requester_pays_project }) # gcloud version 277 and onwards requires you to specify a region. Let's just require it for all hailctl users for consistency. if args.region: project_region = args.region else: project_region = gcloud.get_config("dataproc/region") if not project_region: raise RuntimeError( "Could not determine dataproc region. Use --region argument to hailctl, or use `gcloud config set dataproc/region <my-region>` to set a default." ) # add VEP init script if args.vep: # VEP is too expensive if you have to pay egress charges. We must choose the right replicate. replicate = REGION_TO_REPLICATE_MAPPING.get(project_region) if replicate is None: raise RuntimeError( f"The --vep argument is not currently provided in your region.\n" f" Please contact the Hail team on https://discuss.hail.is for support.\n" f" Your region: {project_region}\n" f" Supported regions: {', '.join(REGION_TO_REPLICATE_MAPPING.keys())}" ) print(f"Pulling VEP data from bucket in {replicate}.") conf.extend_flag('metadata', {"VEP_REPLICATE": replicate}) vep_config_path = "/vep_data/vep-gcloud.json" conf.extend_flag( 'metadata', { "VEP_CONFIG_PATH": vep_config_path, "VEP_CONFIG_URI": f"file://{vep_config_path}" }) conf.extend_flag('initialization-actions', [deploy_metadata[f'vep-{args.vep}.sh']]) # add custom init scripts if args.init: conf.extend_flag('initialization-actions', args.init.split(',')) if args.metadata: conf.parse_and_extend('metadata', args.metadata) wheel = args.wheel or deploy_metadata['wheel'] conf.extend_flag('metadata', {'WHEEL': wheel}) # if Python packages requested, add metadata variable packages = deploy_metadata['pip_dependencies'].strip('|').split('|||') metadata_pkgs = conf.flags['metadata'].get('PKGS') split_regex = r'[|,]' if metadata_pkgs: packages.extend(re.split(split_regex, metadata_pkgs)) if args.packages: packages.extend(re.split(split_regex, args.packages)) conf.extend_flag('metadata', {'PKGS': '|'.join(set(packages))}) def disk_size(size): if args.vep: size = max(size, 200) return str(size) + 'GB' conf.extend_flag( 'properties', { "spark:spark.driver.memory": "{driver_memory}g".format(driver_memory=str( int(MACHINE_MEM[args.master_machine_type] * args.master_memory_fraction))) }) conf.flags['master-machine-type'] = args.master_machine_type conf.flags['master-boot-disk-size'] = '{}GB'.format( args.master_boot_disk_size) conf.flags['num-master-local-ssds'] = args.num_master_local_ssds conf.flags['num-secondary-workers'] = args.num_secondary_workers conf.flags['num-worker-local-ssds'] = args.num_worker_local_ssds conf.flags['num-workers'] = args.num_workers conf.flags['secondary-worker-boot-disk-size'] = disk_size( args.secondary_worker_boot_disk_size) conf.flags['worker-boot-disk-size'] = disk_size(args.worker_boot_disk_size) conf.flags['worker-machine-type'] = args.worker_machine_type if args.region: conf.flags['region'] = args.region if args.zone: conf.flags['zone'] = args.zone conf.flags['initialization-action-timeout'] = args.init_timeout if args.network: conf.flags['network'] = args.network if args.configuration: conf.flags['configuration'] = args.configuration if args.project: conf.flags['project'] = args.project if args.bucket: conf.flags['bucket'] = args.bucket account = gcloud.get_config("account") if account: conf.flags['labels'] = 'creator=' + re.sub(r'[^0-9a-z_\-]', '_', account.lower())[:63] # rewrite metadata and properties to escape them conf.flags['metadata'] = '^|||^' + '|||'.join( f'{k}={v}' for k, v in conf.flags['metadata'].items()) conf.flags['properties'] = '^|||^' + '|||'.join( f'{k}={v}' for k, v in conf.flags['properties'].items()) # command to start cluster cmd = conf.get_command(args.name) if args.beta or args.max_idle or args.max_age: cmd.insert(1, 'beta') if args.max_idle: cmd.append('--max-idle={}'.format(args.max_idle)) if args.max_age: cmd.append('--max-age={}'.format(args.max_age)) if args.expiration_time: cmd.append('--expiration_time={}'.format(args.expiration_time)) cmd.extend(pass_through_args) # print underlying gcloud command print(' '.join(cmd[:5]) + ' \\\n ' + ' \\\n '.join(cmd[5:])) # spin up cluster if not args.dry_run: print("Starting cluster '{}'...".format(args.name)) gcloud.run(cmd[1:]) if args.master_tags: add_tags_command = [ 'compute', 'instances', 'add-tags', args.name + '-m', '--tags', args.master_tags ] if args.project: add_tags_command.append(f"--project={args.project}") if args.zone: add_tags_command.append(f"--zone={args.zone}") print('gcloud ' + ' '.join(add_tags_command)) if not args.dry_run: gcloud.run(add_tags_command)
def get_bond_recent_data(bond, as_json=False, order='ascending', interval='Daily'): """ This function retrieves recent historical data from the introduced bond from Investing.com. So on, the recent data of the introduced bond will be retrieved and returned as a :obj:`pandas.DataFrame` if the parameters are valid and the request to Investing.com succeeds. Note that additionally some optional parameters can be specified: as_json and order, which let the user decide if the data is going to be returned as a :obj:`json` or not, and if the recent data is going to be ordered ascending or descending (where the index is the date), respectively. Args: bond (:obj:`str`): name of the bond to retrieve recent historical data from. as_json (:obj:`bool`, optional): to determine the format of the output data, either a :obj:`pandas.DataFrame` if False and a :obj:`json` if True. order (:obj:`str`, optional): to define the order of the retrieved data which can either be ascending or descending. interval (:obj:`str`, optional): value to define the historical data interval to retrieve, by default `Daily`, but it can also be `Weekly` or `Monthly`. Returns: :obj:`pandas.DataFrame` or :obj:`json`: The function can return either a :obj:`pandas.DataFrame` or a :obj:`json` object, containing the retrieved recent data of the specified bond. So on, the resulting dataframe contains the open, high, low and close values for the selected bond on market days. The resulting recent data, in case that the default parameters were applied, will look like:: Date || Open | High | Low | Close -----||------|------|-----|------- xxxx || xxxx | xxxx | xxx | xxxxx but in case that as_json parameter was defined as True, then the output will be:: { name: name, recent: [ { date: 'dd/mm/yyyy', open: x, high: x, low: x, close: x }, ... ] } Raises: ValueError: raised whenever any of the introduced arguments is not valid or errored. IOError: raised if bonds object/file was not found or unable to retrieve. RuntimeError: raised if the introduced bond was not found or did not match any of the existing ones. ConnectionError: raised if connection to Investing.com could not be established. IndexError: raised if bond historical data was unavailable or not found in Investing.com. Examples: >>> investpy.get_bond_recent_data(bond='Argentina 3Y') Open High Low Close Date 2019-09-23 52.214 52.214 52.214 52.214 2019-09-24 52.323 52.323 52.323 52.323 2019-09-25 52.432 52.432 52.432 52.432 2019-09-26 52.765 52.765 52.765 52.765 2019-09-27 52.876 52.876 52.876 52.876 """ if not bond: raise ValueError("ERR#0066: bond parameter is mandatory and must be a valid bond name.") if not isinstance(bond, str): raise ValueError("ERR#0067: bond argument needs to be a str.") if not isinstance(as_json, bool): raise ValueError("ERR#0002: as_json argument can just be True or False, bool type.") if order not in ['ascending', 'asc', 'descending', 'desc']: raise ValueError("ERR#0003: order argument can just be ascending (asc) or descending (desc), str type.") if not interval: raise ValueError("ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'.") if not isinstance(interval, str): raise ValueError("ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'.") if interval not in ['Daily', 'Weekly', 'Monthly']: raise ValueError("ERR#0073: interval value should be a str type and it can just be either 'Daily', 'Weekly' or 'Monthly'.") resource_package = 'investpy' resource_path = '/'.join(('resources', 'bonds', 'bonds.csv')) if pkg_resources.resource_exists(resource_package, resource_path): bonds = pd.read_csv(pkg_resources.resource_filename(resource_package, resource_path)) else: raise FileNotFoundError("ERR#0064: bonds file not found or errored.") if bonds is None: raise IOError("ERR#0065: bonds object not found or unable to retrieve.") bond = bond.strip() bond = bond.lower() if unidecode.unidecode(bond) not in [unidecode.unidecode(value.lower()) for value in bonds['name'].tolist()]: raise RuntimeError("ERR#0068: bond " + bond + " not found, check if it is correct.") id_ = bonds.loc[(bonds['name'].str.lower() == bond).idxmax(), 'id'] name = bonds.loc[(bonds['name'].str.lower() == bond).idxmax(), 'name'] full_name = bonds.loc[(bonds['name'].str.lower() == bond).idxmax(), 'full_name'] header = full_name + " Bond Yield Historical Data" params = { "curr_id": id_, "smlID": str(randint(1000000, 99999999)), "header": header, "interval_sec": interval, "sort_col": "date", "sort_ord": "DESC", "action": "historical_data" } head = { "User-Agent": get_random(), "X-Requested-With": "XMLHttpRequest", "Accept": "text/html", "Accept-Encoding": "gzip, deflate, br", "Connection": "keep-alive", } url = "https://www.investing.com/instruments/HistoricalDataAjax" req = requests.post(url, headers=head, data=params) if req.status_code != 200: raise ConnectionError("ERR#0015: error " + str(req.status_code) + ", try again later.") root_ = fromstring(req.text) path_ = root_.xpath(".//table[@id='curr_table']/tbody/tr") result = list() if path_: for elements_ in path_: if elements_.xpath(".//td")[0].text_content() == 'No results found': raise IndexError("ERR#0069: bond information unavailable or not found.") info = [] for nested_ in elements_.xpath(".//td"): info.append(nested_.get('data-real-value')) bond_date = datetime.fromtimestamp(int(info[0])) bond_date = date(bond_date.year, bond_date.month, bond_date.day) bond_close = float(info[1].replace(',', '')) bond_open = float(info[2].replace(',', '')) bond_high = float(info[3].replace(',', '')) bond_low = float(info[4].replace(',', '')) result.insert(len(result), Data(bond_date, bond_open, bond_high, bond_low, bond_close, None, None, None)) if order in ['ascending', 'asc']: result = result[::-1] elif order in ['descending', 'desc']: result = result if as_json is True: json_ = { 'name': name, 'recent': [value.bond_as_json() for value in result] } return json.dumps(json_, sort_keys=False) elif as_json is False: df = pd.DataFrame.from_records([value.bond_to_dict() for value in result]) df.set_index('Date', inplace=True) return df else: raise RuntimeError("ERR#0004: data retrieval error while scraping.")
# but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import sys import pkg_resources as pr import os import subprocess # Initialize binaries if sys.platform[0:5] == 'linux': assert (pr.resource_exists('lomap', 'binaries/linux/ltl2ba')) assert (pr.resource_exists('lomap', 'binaries/linux/scheck2')) ltl2ba_binary = pr.resource_filename('lomap', 'binaries/linux/ltl2ba') scheck_binary = pr.resource_filename('lomap', 'binaries/linux/scheck2') elif sys.platform == 'darwin': assert (pr.resource_exists('lomap', 'binaries/mac/ltl2ba')) assert (pr.resource_exists('lomap', 'binaries/mac/scheck2')) ltl2ba_binary = pr.resource_filename('lomap', 'binaries/mac/ltl2ba') scheck_binary = pr.resource_filename('lomap', 'binaries/mac/scheck2') else: print >> sys.stderr, sys.platform, 'platform not supported yet!' print >> sys.stderr, 'Binaries will not work!' ltl2ba_binary = None scheck_binary = None # Check and fix chmod as needed
def currency_crosses_as_dict(base=None, second=None, columns=None, as_json=False): """ This function retrieves all the available currency crosses from Investing.com and returns them as a :obj:`dict`, which contains not just the currency crosses names, but all the fields contained on the currency_crosses file is columns is None, otherwise, just the specified column values will be returned. Note that the filtering params are both base and second, which mean the base and the second currency of the currency cross, for example, in the currency cross `EUR/USD` the base currency is EUR and the second currency is USD. These are optional parameters, so specifying one of them means that all the currency crosses where the introduced currency is either base or second will be returned; if both are specified, just the introduced currency cross will be returned if it exists. All the available currency crosses can be found at: https://www.investing.com/currencies/ Args: base (:obj:`str`, optional): symbol of the base currency of the currency cross, this will return a :obj:`pandas.DataFrame` containing all the currency crosses where the base currency matches the introduced one. second (:obj:`str`): symbol of the second currency of the currency cross, this will return a :obj:`pandas.DataFrame` containing all the currency crosses where the second currency matches the introduced one. columns (:obj:`list`, optional): names of the columns of the currency crosses data to retrieve <name, full_name, base, base_name, second, second_name> as_json (:obj:`bool`, optional): value to determine the format of the output data which can either be a :obj:`dict` or a :obj:`json`. Returns: :obj:`dict` or :obj:`json` - currency_crosses_dict: The resulting :obj:`dict` contains the retrieved data if found, if not, the corresponding fields are filled with `None` values. In case the information was successfully retrieved, the :obj:`dict` will look like:: currency_crosses_dict = { 'name': name, 'full_name': full_name, 'base': base, 'base_name': base_name, 'second': second, 'second_name': second_name } Raises: ValueError: raised if any of the introduced arguments is not valid or errored. FileNotFoundError: raised if `currency_crosses.csv` file was not found. IOError: raised if currency crosses retrieval failed, both for missing file or empty file. """ if base is not None and not isinstance(base, str): raise ValueError( "ERR#0049: specified base currency value is not valid.") if second is not None and not isinstance(second, str): raise ValueError( "ERR#0051: specified second currency value is not valid.") if not isinstance(as_json, bool): raise ValueError( "ERR#0002: as_json argument can just be True or False, bool type.") resource_package = 'investpy' resource_path = '/'.join(('resources', 'currency_crosses.csv')) if pkg_resources.resource_exists(resource_package, resource_path): currency_crosses = pd.read_csv(pkg_resources.resource_filename( resource_package, resource_path), keep_default_na=False) else: raise FileNotFoundError( "ERR#0060: currency_crosses file not found or errored.") if currency_crosses is None: raise IOError( "ERR#0050: currency_crosses not found or unable to retrieve.") available_currencies = available_currencies_as_list() currency_crosses.drop(columns=['tag', 'id'], inplace=True) currency_crosses = currency_crosses.where(pd.notnull(currency_crosses), None) if columns is None: columns = currency_crosses.columns.tolist() else: if not isinstance(columns, list): raise ValueError( "ERR#0020: specified columns argument is not a list, it can just be list type." ) if not all(column in currency_crosses.columns.tolist() for column in columns): raise ValueError( "ERR#0021: specified columns does not exist, available columns are " "<name, full_name, base, base_name, second, second_name>") if base is None and second is None: currency_crosses.reset_index(drop=True, inplace=True) if as_json: return json.dumps( currency_crosses[columns].to_dict(orient='records')) else: return currency_crosses[columns].to_dict(orient='records') elif base is not None: base = unidecode(base.strip().upper()) if base not in available_currencies: raise ValueError("ERR#0053: the introduced currency " + base + " does not exists.") if second is not None: second = unidecode(second.strip().upper()) if second not in available_currencies: raise ValueError("ERR#0053: the introduced currency " + second + " does not exists.") currency_crosses = currency_crosses[ (currency_crosses['base'] == base) & (currency_crosses['second'] == second)] currency_crosses.reset_index(drop=True, inplace=True) if len(currency_crosses) > 0: if as_json: return json.dumps( currency_crosses[columns].to_dict(orient='records')) else: return currency_crosses[columns].to_dict(orient='records') else: raise ValueError("ERR#0054: the introduced currency cross " + base + "/" + second + " does not exists.") else: currency_crosses = currency_crosses[currency_crosses['base'] == base] currency_crosses.reset_index(drop=True, inplace=True) if as_json: return json.dumps( currency_crosses[columns].to_dict(orient='records')) else: return currency_crosses[columns].to_dict(orient='records') elif second is not None: second = unidecode(second.strip().upper()) if second not in available_currencies: raise ValueError("ERR#0053: the introduced currency " + second + " does not exists.") currency_crosses = currency_crosses[currency_crosses['second'] == unidecode(second.upper())] currency_crosses.reset_index(drop=True, inplace=True) if as_json: return json.dumps( currency_crosses[columns].to_dict(orient='records')) else: return currency_crosses[columns].to_dict(orient='records')
def _has_resource(self, resource): return pkg_resources.resource_exists(__name__, resource.name)
def __init__(self, params, status, calib_data, service_name): """ :param params: pywws configuration. :type params: :class:`pywws.DataStore.params` :param status: pywws status store. :type status: :class:`pywws.DataStore.status` :param calib_data: 'calibrated' data. :type calib_data: :class:`pywws.DataStore.calib_store` :param service_name: name of service to upload to. :type service_name: string """ self.logger = logging.getLogger('pywws.ToService(%s)' % service_name) self.params = params self.status = status self.data = calib_data self.service_name = service_name # 'derived' services such as 'underground_rf' share their # parent's config and templates config_section = self.service_name.split('_')[0] if config_section == self.service_name: self.parent = None else: self.parent = config_section self.old_response = None self.old_ex = None # set default socket timeout, so urlopen calls don't hang forever socket.setdefaulttimeout(30) # open params file service_params = SafeConfigParser() service_params.optionxform = str service_params.readfp( pkg_resources.resource_stream( 'pywws', 'services/%s.ini' % (self.service_name))) # get URL self.server = service_params.get('config', 'url') parsed_url = urlparse.urlsplit(self.server) if parsed_url.scheme == 'aprs': self.send_data = self.aprs_send_data server, port = parsed_url.netloc.split(':') self.server = (server, int(port)) elif parsed_url.scheme == 'mqtt': self.send_data = self.mqtt_send_data else: self.send_data = self.http_send_data self.use_get = eval(service_params.get('config', 'use get')) # get fixed part of upload data self.fixed_data = dict() for name, value in service_params.items('fixed'): if value[0] == '*': value = self.params.get(config_section, value[1:], 'unknown') self.fixed_data[name] = value # create templater self.templater = Template.Template(self.params, self.status, self.data, self.data, None, None, use_locale=False) template_name = 'services/%s_template_%s.txt' % ( config_section, self.params.get('config', 'ws type')) if not pkg_resources.resource_exists('pywws', template_name): template_name = 'services/%s_template_1080.txt' % (config_section) self.template_file = pkg_resources.resource_stream( 'pywws', template_name) # get other parameters self.auth_type = service_params.get('config', 'auth_type') if self.auth_type == 'basic': user = self.params.get(config_section, 'user', 'unknown') password = self.params.get(config_section, 'password', 'unknown') self.auth = 'Basic %s' % base64.b64encode('%s:%s' % (user, password)) self.catchup = eval(service_params.get('config', 'catchup')) self.expected_result = eval(service_params.get('config', 'result')) self.interval = eval(service_params.get('config', 'interval')) self.interval = max(self.interval, 40) self.interval = timedelta(seconds=self.interval) # move 'last update' from params to status last_update = self.params.get_datetime(self.service_name, 'last update') if last_update: self.params.unset(self.service_name, 'last update') self.status.set('last update', self.service_name, last_update.isoformat(' ')) # set timestamp of first data to upload self.next_update = datetime.utcnow() - max( timedelta(days=self.catchup), self.interval)
def currency_crosses_as_df(base=None, second=None): """ This function retrieves all the available currency crosses from Investing.com and returns them as a :obj:`pandas.DataFrame`, which contains not just the currency crosses names, but all the fields contained on the currency_crosses file. Note that the filtering params are both base and second, which mean the base and the second currency of the currency cross, for example, in the currency cross `EUR/USD` the base currency is EUR and the second currency is USD. These are optional parameters, so specifying one of them means that all the currency crosses where the introduced currency is either base or second will be returned; if both are specified, just the introduced currency cross will be returned if it exists. All the available currency crosses can be found at: https://www.investing.com/currencies/ Args: base (:obj:`str`, optional): symbol of the base currency of the currency cross, this will return a :obj:`pandas.DataFrame` containing all the currency crosses where the base currency matches the introduced one. second (:obj:`str`): symbol of the second currency of the currency cross, this will return a :obj:`pandas.DataFrame` containing all the currency crosses where the second currency matches the introduced one. Returns: :obj:`pandas.DataFrame` - currency_crosses_df: The resulting :obj:`pandas.DataFrame` contains all the currency crosses basic information retrieved from Investing.com. In case the information was successfully retrieved, the resulting :obj:`pandas.DataFrame` will look like:: name | full_name | base | second | base_name | second_name -----|-----------|------|--------|-----------|------------- xxxx | xxxxxxxxx | xxxx | xxxxxx | xxxxxxxxx | xxxxxxxxxxx Raises: ValueError: raised if any of the introduced arguments is not valid or errored. FileNotFoundError: raised if `currency_crosses.csv` file was not found. IOError: raised if currency crosses retrieval failed, both for missing file or empty file. """ if base is not None and not isinstance(base, str): raise ValueError( "ERR#0049: specified base currency value is not valid.") if second is not None and not isinstance(second, str): raise ValueError( "ERR#0051: specified second currency value is not valid.") resource_package = 'investpy' resource_path = '/'.join(('resources', 'currency_crosses.csv')) if pkg_resources.resource_exists(resource_package, resource_path): currency_crosses = pd.read_csv(pkg_resources.resource_filename( resource_package, resource_path), keep_default_na=False) else: raise FileNotFoundError( "ERR#0060: currency_crosses file not found or errored.") if currency_crosses is None: raise IOError( "ERR#0050: currency_crosses not found or unable to retrieve.") available_currencies = available_currencies_as_list() currency_crosses.drop(columns=['tag', 'id'], inplace=True) currency_crosses = currency_crosses.where(pd.notnull(currency_crosses), None) if base is None and second is None: currency_crosses.reset_index(drop=True, inplace=True) return currency_crosses elif base is not None: base = unidecode(base.strip().upper()) if base not in available_currencies: raise ValueError("ERR#0053: the introduced currency " + base + " does not exists.") if second is not None: second = unidecode(second.strip().upper()) if second not in available_currencies: raise ValueError("ERR#0053: the introduced currency " + second + " does not exists.") currency_crosses = currency_crosses[ (currency_crosses['base'] == base) & (currency_crosses['second'] == second)] currency_crosses.reset_index(drop=True, inplace=True) if len(currency_crosses) > 0: return currency_crosses else: raise ValueError("ERR#0054: the introduced currency cross " + base + "/" + second + " does not exists.") else: currency_crosses = currency_crosses[currency_crosses['base'] == base] currency_crosses.reset_index(drop=True, inplace=True) return currency_crosses elif second is not None: second = unidecode(second.strip().upper()) if second not in available_currencies: raise ValueError("ERR#0053: the introduced currency " + second + " does not exists.") currency_crosses = currency_crosses[currency_crosses['second'] == second] currency_crosses.reset_index(drop=True, inplace=True) return currency_crosses
from logging import getLogger from ConfigParser import ConfigParser from locale import getdefaultlocale from pkg_resources import resource_filename, resource_exists from codecs import open try: from ..comm import notif except ValueError: from comm import notif try: from ..arping import arping, arpingerror except ValueError: from arping import arping, arpingerror # Set language messages to use if resource_exists("homenetcontrol", "langs/" + getdefaultlocale()[0][:2] + ".lang"): langfile = resource_filename( "homenetcontrol", "langs/" + getdefaultlocale()[0][:2] + ".lang") else: langfile = resource_filename("homenetcontrol", "langs/en.lang") lang = ConfigParser() lang.readfp(open(langfile, "r", "utf8")) # Get info from config file if resource_exists("homenetcontrol", "check_devices/config.ini"): configfile = resource_filename("homenetcontrol", "check_devices/config.ini") else: configfile = "/etc/homenetcontrol/config.ini" config = ConfigParser() config.optionxform = str
def get_commodity_information(commodity, country=None, as_json=False): """ This function retrieves fundamental financial information from the specified commodity. The retrieved information from the commodity can be valuable as it is additional information that can be used combined with OHLC values, so to determine financial insights from the company which holds the specified commodity. Args: commodity (:obj:`str`): name of the commodity to retrieve information from. country (:obj:`str`, optional): name of the country to retrieve the commodity information from (if there is more than one country that provides data from the same commodity). as_json (:obj:`bool`, optional): optional argument to determine the format of the output data (:obj:`dict` or :obj:`json`). Returns: :obj:`pandas.DataFrame` or :obj:`dict`- commodity_information: The resulting :obj:`pandas.DataFrame` contains the information fields retrieved from Investing.com from the specified commodity; it can also be returned as a :obj:`dict`, if argument `as_json=True`. If any of the information fields could not be retrieved, that field/s will be filled with None values. If the retrieval process succeeded, the resulting :obj:`dict` will look like:: commodity_information = { "1-Year Change": "16.15%", "52 wk Range": "1,270.2 - 1,566.2", "Base Symbol": "GC", "Commodity Name": "Gold", "Contract Size": "100 Troy Ounces", "Last Rollover Day": "24/11/2019", "Month": "Feb 20", "Months": "GJMQVZ", "Open": 1479.8, "Point Value": "$100", "Prev. Close": 1481.2, "Settlement Day": "25/01/2020", "Settlement Type": "Physical", "Tick Size": 0.1, "Tick Value": 10.0, "Todays Range": "1,477.55 - 1,484.25" } Raises: ValueError: raised if any of the introduced arguments is not valid or errored. FileNotFoundError: raised if `commodities.csv` file was not found or errored. IOError: raised if commodities.csv file is empty or errored. RuntimeError: raised if scraping process failed while running. ConnectionError: raised if the connection to Investing.com errored (did not return HTTP 200) """ if not commodity: raise ValueError( "ERR#0078: commodity parameter is mandatory and must be a valid commodity name." ) if not isinstance(commodity, str): raise ValueError( "ERR#0078: commodity parameter is mandatory and must be a valid commodity name." ) if country is not None and not isinstance(country, str): raise ValueError("ERR#0025: specified country value not valid.") if not isinstance(as_json, bool): raise ValueError( "ERR#0002: as_json argument can just be True or False, bool type.") resource_package = 'investpy' resource_path = '/'.join(('resources', 'commodities.csv')) if pkg_resources.resource_exists(resource_package, resource_path): commodities = pd.read_csv( pkg_resources.resource_filename(resource_package, resource_path)) else: raise FileNotFoundError( "ERR#0075: commodities file not found or errored.") if commodities is None: raise IOError("ERR#0076: commodities not found or unable to retrieve.") commodity = commodity.strip() commodity = commodity.lower() if unidecode(commodity) not in [ unidecode(value.lower()) for value in commodities['name'].tolist() ]: raise RuntimeError("ERR#0079: commodity " + commodity + " not found, check if it is correct.") if country is None: found_commodities = commodities[commodities['name'].str.lower() == commodity] if len(found_commodities) > 1: msg = "Note that the displayed commodity information can differ depending on the country. " \ "If you want to retrieve " + commodity + " data from either " + \ " or ".join(found_commodities['country'].tolist()) + ", specify the country parameter." warnings.warn(msg, Warning) del found_commodities else: if unidecode(country.lower()) not in commodities['country'].unique( ).tolist(): raise RuntimeError("ERR#0034: country " + country.lower() + " not found, check if it is correct.") commodities = commodities[commodities['country'] == unidecode( country.lower())] name = commodities.loc[( commodities['name'].str.lower() == commodity).idxmax(), 'name'] tag = commodities.loc[( commodities['name'].str.lower() == commodity).idxmax(), 'tag'] url = "https://www.investing.com/commodities/" + tag head = { "User-Agent": random_user_agent(), "X-Requested-With": "XMLHttpRequest", "Accept": "text/html", "Accept-Encoding": "gzip, deflate, br", "Connection": "keep-alive", } req = requests.get(url, headers=head) if req.status_code != 200: raise ConnectionError("ERR#0015: error " + str(req.status_code) + ", try again later.") root_ = fromstring(req.text) path_ = root_.xpath("//div[contains(@class, 'overviewDataTable')]/div") result = pd.DataFrame(columns=[ 'Commodity Name', 'Prev. Close', 'Month', 'Tick Size', 'Open', 'Contract Size', 'Tick Value', 'Todays Range', 'Settlement Type', 'Base Symbol', '52 wk Range', 'Settlement Day', 'Point Value', '1-Year Change', 'Last Rollover Day', 'Months' ]) result.at[0, 'Commodity Name'] = name if path_: for elements_ in path_: element = elements_.xpath(".//span[@class='float_lang_base_1']")[0] title_ = element.text_content() if title_ == "Day's Range": title_ = 'Todays Range' if title_ in result.columns.tolist(): try: result.at[0, title_] = float( element.getnext().text_content().replace(',', '')) continue except: pass try: text = element.getnext().text_content().strip() result.at[0, title_] = datetime.strptime( text, "%m/%d/%Y").strftime("%d/%m/%Y") continue except: pass try: text = element.getnext().text_content().strip() if text.__contains__('1 = '): result.at[0, title_] = text.replace('1 = ', '') continue except: pass try: value = element.getnext().text_content().strip() if value.__contains__('K'): value = float(value.replace('K', '').replace(',', '')) * 1e3 elif value.__contains__('M'): value = float(value.replace('M', '').replace(',', '')) * 1e6 elif value.__contains__('B'): value = float(value.replace('B', '').replace(',', '')) * 1e9 elif value.__contains__('T'): value = float(value.replace('T', '').replace( ',', '')) * 1e12 result.at[0, title_] = value continue except: pass result.replace({'N/A': None}, inplace=True) if as_json is True: json_ = result.iloc[0].to_dict() return json_ elif as_json is False: return result else: raise RuntimeError("ERR#0004: data retrieval error while scraping.")