def test_locate_success(self, test_fs: tempfile.TemporaryDirectory,
path_list: List[str], expected_result: str):
root: Path = Path(test_fs.name)
dir: FileRepoDir = FileRepoDir(root)
obj_path: RepoObjectPath = RepoObjectPath.from_object(
dir.get(path_list), self.REPO_FORMAT)
assert str(obj_path) == expected_result
def new_object(self, obj_path: str, obj_name: str) -> RepoObject:
"""
Creates a new object at the provided path
Parameters
----------
obj_path: str
the path to the object
obj_name: str
the object name
Returns
-------
RepoObject
the newly created repo object
"""
obj_path: RepoObjectPath = RepoObjectPath.from_uri(
obj_path, self.__format)
cur_dir: RepoEntity = self.__root
for name in obj_path:
if name not in cur_dir:
cur_dir = cur_dir.new_dir(name)
else:
cur_dir = cur_dir[name]
return cur_dir.new_object(obj_name)
def iterate_missing(self, from_date: Date,
to_date: Date) -> Iterator[RepoURI]:
"""
Identifies objects that are not in the repository
or need to be updated for the given dates
Parameters
----------
from_date: Date
the start date
to_date: Date
the end date
Returns
-------
Iterator[str]
an iterator for missing objects
"""
self.refresh()
track_year, track_quarter = 0, 0
cur_holidays: us_holidays = None
cur_date: Date = from_date.copy()
for _ in range(to_date.diff_days(from_date)):
(cur_year, cur_quarter, *_) = cur_date.tuple()
if cur_year != track_year:
# Moving to the first or to the next year
cur_holidays = us_holidays(cur_year)
track_year, track_quarter = cur_year, 0
if not (cur_date.is_weekend() or cur_date in cur_holidays):
obj_path: RepoObjectPath = RepoObjectPath.from_date(
DatePeriodType.DAY, cur_date, self.__format)
if str(obj_path) not in self.__index:
if cur_quarter != track_quarter:
# Add a quartely file to the update list
# only if it has not been added before
yield RepoObjectPath.from_date(DatePeriodType.QUARTER,
cur_date, self.__format)
track_quarter = cur_quarter
# Add a daily file to the update list
yield obj_path
# next date
cur_date += 1
def test_getitem(self, path: List[str], date_period: str, quarter: str,
year: int, date_str: str) -> None:
obj_path: RepoObjectPath = RepoObjectPath.from_list(
path, self.REPO_FORMAT)
assert obj_path[0] == date_period
assert obj_path[1] == year
assert obj_path[2] == quarter
assert obj_path[3] == Date(date_str).format(
'master{y}{m:02}{d:02}.idx')
def test_init_with_list(self, path: List[str], date_period: str,
quarter: int, year: int, date_str: str) -> None:
obj_path: RepoObjectPath = RepoObjectPath.from_list(
path, self.REPO_FORMAT)
assert obj_path.date_period_type() == DatePeriodType.from_string(
date_period)
assert obj_path.year() == year
assert obj_path.quarter() == quarter
assert obj_path.date() == Date(date_str)
def find(self, period_type: DatePeriodType, the_date: Date) -> RepoObject:
"""
Finds an object for the given date and period type
Parameters
----------
period_type: DatePeriodType
the date period type
the_date: Date
the date
Returns
-------
RepoObject
the object
"""
return self.get_object(
str(RepoObjectPath.from_date(period_type, the_date,
self.__format)))
def create(self, period_type: DatePeriodType,
the_date: Date) -> RepoObject:
"""
Creates an object for the given date and period type
Parameters
----------
period_type: DatePeriodType
the period type
the_date: Date
the date
Returns
-------
RepoObject
"""
obj_path: RepoObjectPath = RepoObjectPath.from_date(
period_type, the_date, self.__format)
return self.new_object(obj_path.parent(), obj_path[-1])
def test_find_missing(self, edgar_fs: tempfile.TemporaryDirectory,
repo_format: RepoFormat):
root: Path = Path(edgar_fs.name)
fs: FileRepoFS = FileRepoFS(root, repo_format)
holidays_sample: List[Date] = [
Date('2018-01-01'),
Date('2018-01-15'),
Date('2018-02-19'),
Date('2018-05-13'),
Date('2018-05-28'),
Date('2018-07-04'),
Date('2018-09-03'),
Date('2018-10-08'),
Date('2018-11-12'),
Date('2018-11-22'),
Date('2018-12-25'),
Date('2019-01-01'),
Date('2019-01-21'),
Date('2019-02-18'),
Date('2019-05-27'),
]
q: int = 0
d: int = 0
for i in fs.find_missing(Date('2017-09-10'), Date('2019-05-25')):
obj_path: RepoObjectPath = RepoObjectPath.from_uri(i, repo_format)
if obj_path[0] == str(DatePeriodType.QUARTER):
assert obj_path[-1] == 'master.idx'
q += 1
else:
the_date: Date = obj_path.date()
assert not the_date.is_weekend()
assert the_date not in holidays_sample
d += 1
assert q == 7
assert d == 350
def get_object(self, obj_uri: str) -> RepoObject:
"""
Get a repo object at the given relative path
Parameters
----------
obj_uri: str
the object URI
Returns
-------
RepoObject | None
the repo objet at the given path. If no object is found then None is returned
"""
obj_path: RepoObjectPath = RepoObjectPath.from_uri(
obj_uri, self.__format)
cur_ent: RepoEntity = self.__root
for i in obj_path:
if i in cur_ent:
cur_ent = cur_ent[i]
else:
return None
return cur_ent
def test_str(self, path: List[str], expected_result: str) -> None:
obj_path: RepoObjectPath = RepoObjectPath.from_list(
path, self.REPO_FORMAT)
assert str(obj_path) == expected_result
def test_parent_success(self, path: List[str], parent: str):
obj_path: RepoObjectPath = RepoObjectPath.from_list(
path, self.REPO_FORMAT)
assert obj_path.parent() == parent
def visit(self, obj: RepoObject) -> bool:
obj_path: RepoObjectPath = RepoObjectPath.from_object(
obj, self.__format)
self.__index[str(obj_path)] = obj
return True
def missing(repo_format: RepoFormat) -> Iterator[RepoObjectPath]:
return iter([
RepoObjectPath.from_date(DatePeriodType.DAY, Date('2021-07-12'), repo_format),
RepoObjectPath.from_date(DatePeriodType.DAY, Date('2021-07-13'), repo_format),
RepoObjectPath.from_date(DatePeriodType.DAY, Date('2021-07-14'), repo_format),
])