def iter_doc_objs(self, **kwargs):
"""Returns a pair: (doc_id, nexson_blob)
for each document in this repository.
Order is arbitrary.
"""
_LOG = get_logger("TypeAwareGitShard")
try:
for doc_id, fp in self.iter_doc_filepaths(**kwargs):
if not self._is_alias(doc_id):
# TODO:hook for type-specific parser?
with codecs.open(fp, "r", "utf-8") as fo:
try:
nex_obj = anyjson.loads(fo.read())
yield (doc_id, nex_obj)
except Exception:
pass
except Exception as x:
f = "iter_doc_filepaths FAILED with this error:\n{}"
f = f.format(str(x))
_LOG.warn(f)
def iter_doc_objs(self, **kwargs):
"""Returns a pair: (doc_id, nexson_blob)
for each document in this repository.
Order is arbitrary.
"""
_LOG = get_logger('TypeAwareGitShard')
try:
for doc_id, fp in self.iter_doc_filepaths(**kwargs):
if not self._is_alias(doc_id):
# TODO:hook for type-specific parser?
with codecs.open(fp, 'r', 'utf-8') as fo:
try:
nex_obj = anyjson.loads(fo.read())
yield (doc_id, nex_obj)
except Exception:
pass
except Exception as x:
f = 'iter_doc_filepaths FAILED with this error:\n{}'
f = f.format(str(x))
_LOG.warn(f)
"""Base class git action manager (subclasses will accommodate each type)"""
from peyotl.utility.str_util import is_str_type
from peyotl.nexson_syntax import write_as_json
from peyotl.utility import get_logger
import os
# noinspection PyUnresolvedReferences
from sh import git # pylint: disable=E0611
import shutil
import sh
import locket
import codecs
import tempfile # @TEMPORARY for deprecated write_study
_LOG = get_logger(__name__)
def get_HEAD_SHA1(git_dir):
"""Not locked!
"""
head_file = os.path.join(git_dir, 'HEAD')
with open(head_file, 'r') as hf:
head_contents = hf.read().strip()
assert head_contents.startswith('ref: ')
ref_filename = head_contents[5:] # strip off "ref: "
real_ref = os.path.join(git_dir, ref_filename)
with open(real_ref, 'r') as rf:
return rf.read().strip()
def get_user_author(auth_info):
"""Return commit author info from a dict. Returns username and author string.
def __init__(
self,
name,
path,
doc_holder_subpath="",
assumed_doc_version=None,
detect_doc_version_fn=None,
refresh_doc_index_fn=None,
git_ssh=None,
pkey=None,
git_action_class=None,
push_mirror_repo_path=None,
infrastructure_commit_author="OpenTree API <*****@*****.**>",
**kwargs
):
GitShard.__init__(self, name)
self.filepath_for_doc_id_fn = None # overwritten in refresh_doc_index_fn
self.id_alias_list_fn = None # overwritten in refresh_doc_index_fn
self._infrastructure_commit_author = infrastructure_commit_author
self._locked_refresh_doc_index = refresh_doc_index_fn
self._master_branch_repo_lock = Lock()
self._ga_class = git_action_class
self.git_ssh = git_ssh
self.pkey = pkey
path = os.path.abspath(path)
dot_git = os.path.join(path, ".git")
doc_dir = os.path.join(path, doc_holder_subpath) # type-specific, e.g. 'study'
if not os.path.isdir(path):
raise FailedShardCreationError('"{p}" is not a directory'.format(p=path))
if not os.path.isdir(dot_git):
raise FailedShardCreationError('"{p}" is not a directory'.format(p=dot_git))
if not os.path.isdir(doc_dir):
raise FailedShardCreationError('"{p}" is not a directory'.format(p=doc_dir))
self.path = path
self.doc_dir = doc_dir
with self._index_lock:
self._locked_refresh_doc_index(self, initializing=True)
self.parent_path = os.path.split(path)[0] + "/"
self.git_dir = dot_git
self.push_mirror_repo_path = push_mirror_repo_path
if assumed_doc_version is None:
_LOG = get_logger("TypeAwareGitShard")
try:
# pass this shard to a type-specific test
assumed_doc_version = detect_doc_version_fn(self)
except IndexError as x:
# no documents in this shard!
_LOG.warn("No documents in this shard! Auto-detection of assumed_doc_version failed.")
except Exception as x:
f = "Auto-detection of assumed_doc_version FAILED with this error:\n{}"
f = f.format(str(x))
_LOG.warn(f)
except:
pass
max_file_size = kwargs.get("max_file_size")
if max_file_size is None:
if max_file_size is not None:
try:
max_file_size = int(max_file_size)
except:
m = 'Configuration-base value of max_file_size was "{}". Expecting an integer.'
m = m.format(max_file_size)
raise RuntimeError(m)
self.max_file_size = max_file_size
self.assumed_doc_version = assumed_doc_version
self._known_prefixes = None
def __init__(
self,
name,
path,
doc_holder_subpath='',
assumed_doc_version=None,
detect_doc_version_fn=None,
refresh_doc_index_fn=None,
git_ssh=None,
pkey=None,
git_action_class=None,
push_mirror_repo_path=None,
infrastructure_commit_author='OpenTree API <*****@*****.**>',
**kwargs):
GitShard.__init__(self, name)
self.filepath_for_doc_id_fn = None # overwritten in refresh_doc_index_fn
self.id_alias_list_fn = None # overwritten in refresh_doc_index_fn
self._infrastructure_commit_author = infrastructure_commit_author
self._locked_refresh_doc_index = refresh_doc_index_fn
self._master_branch_repo_lock = Lock()
self._ga_class = git_action_class
self.git_ssh = git_ssh
self.pkey = pkey
path = os.path.abspath(path)
dot_git = os.path.join(path, '.git')
doc_dir = os.path.join(
path, doc_holder_subpath) # type-specific, e.g. 'study'
if not os.path.isdir(path):
raise FailedShardCreationError(
'"{p}" is not a directory'.format(p=path))
if not os.path.isdir(dot_git):
raise FailedShardCreationError(
'"{p}" is not a directory'.format(p=dot_git))
if not os.path.isdir(doc_dir):
raise FailedShardCreationError(
'"{p}" is not a directory'.format(p=doc_dir))
self.path = path
self.doc_dir = doc_dir
with self._index_lock:
self._locked_refresh_doc_index(self, initializing=True)
self.parent_path = os.path.split(path)[0] + '/'
self.git_dir = dot_git
self.push_mirror_repo_path = push_mirror_repo_path
if assumed_doc_version is None:
_LOG = get_logger('TypeAwareGitShard')
try:
# pass this shard to a type-specific test
assumed_doc_version = detect_doc_version_fn(self)
except IndexError as x:
# no documents in this shard!
_LOG.warn(
'No documents in this shard! Auto-detection of assumed_doc_version failed.'
)
except Exception as x:
f = 'Auto-detection of assumed_doc_version FAILED with this error:\n{}'
f = f.format(str(x))
_LOG.warn(f)
except:
pass
max_file_size = kwargs.get('max_file_size')
if max_file_size is None:
if max_file_size is not None:
try:
max_file_size = int(max_file_size)
except:
m = 'Configuration-base value of max_file_size was "{}". Expecting an integer.'
m = m.format(max_file_size)
raise RuntimeError(m)
self.max_file_size = max_file_size
self.assumed_doc_version = assumed_doc_version
self._known_prefixes = None
#!/usr/bin/env python
from peyotl.utility import get_logger, ConfigWrapper
from peyotl.ott import OTT
import subprocess
import sys
import os
_LOG = get_logger('clipeyotl')
out = sys.stdout
def parse_config_file(fp):
try:
# noinspection PyCompatibility
from ConfigParser import SafeConfigParser
except ImportError:
# noinspection PyCompatibility
from configparser import ConfigParser as SafeConfigParser
if not os.path.exists(fp):
raise RuntimeError('The config filepath "{fp}" does not exist.'.format(fp=fp))
config_obj = SafeConfigParser()
config_obj.read(fp)
return config_obj
def config_command(args):
if args.action.lower() == 'list':
fp = args.filepath
if fp:
fp = os.path.abspath(fp)
cfg = parse_config_file(fp)
