def _update(self, provider):
self.provider = provider
self.path = provider.get('config', {}).get('path', None)
if not self.path:
logger.info('No path')
return []
for filename in get_sorted_files(self.path, sort_by=FileSortAttributes.created):
try:
filepath = os.path.join(self.path, filename)
if os.path.isfile(filepath):
stat = os.lstat(filepath)
last_updated = datetime.fromtimestamp(stat.st_mtime, tz=utc)
if self.is_latest_content(last_updated, provider.get('last_updated')):
item = self.parser.parse_file(filepath, self)
self.move_file(self.path, filename, provider=provider, success=True)
yield [item]
else:
self.move_file(self.path, filename, provider=provider, success=True)
except ParserError.ZCZCParserError as ex:
logger.exception("Ingest Type: Teletype - File: {0} could not be processed".format(filename))
self.move_file(self.path, filename, provider=provider, success=False)
raise ParserError.ZCZCParserError(ex, provider)
except ParserError as ex:
self.move_file(self.path, filename, provider=provider, success=False)
except Exception as ex:
self.move_file(self.path, filename, provider=provider, success=False)
raise ProviderError.ingestError(ex, provider)
def _update(self, provider):
self.provider = provider
self.path = provider.get('config', {}).get('path', None)
if not self.path:
return
for filename in get_sorted_files(self.path, sort_by=FileSortAttributes.created):
try:
if os.path.isfile(os.path.join(self.path, filename)):
filepath = os.path.join(self.path, filename)
stat = os.lstat(filepath)
last_updated = datetime.fromtimestamp(stat.st_mtime, tz=utc)
if self.is_latest_content(last_updated, provider.get('last_updated')):
with open(os.path.join(self.path, filename), 'r') as f:
item = self.parser.parse_message(etree.fromstring(f.read()), provider)
self.add_timestamps(item)
self.move_file(self.path, filename, provider=provider, success=True)
yield [item]
else:
self.move_file(self.path, filename, provider=provider, success=True)
except etreeParserError as ex:
logger.exception("Ingest Type: AFP - File: {0} could not be processed".format(filename), ex)
self.move_file(self.path, filename, provider=provider, success=False)
raise ParserError.newsmlOneParserError(ex, provider)
except ParserError as ex:
self.move_file(self.path, filename, provider=provider, success=False)
except Exception as ex:
self.move_file(self.path, filename, provider=provider, success=False)
raise ProviderError.ingestError(ex, provider)
push_notification('ingest:update')
def _update(self, provider):
self.provider = provider
self.path = provider.get('config', {}).get('path', None)
if not self.path:
return
for filename in get_sorted_files(self.path, sort_by=FileSortAttributes.created):
try:
if os.path.isfile(os.path.join(self.path, filename)):
filepath = os.path.join(self.path, filename)
stat = os.lstat(filepath)
last_updated = datetime.fromtimestamp(stat.st_mtime, tz=utc)
if self.is_latest_content(last_updated, provider.get('last_updated')):
with open(os.path.join(self.path, filename), 'r') as f:
item = self.parser.parse_message(etree.fromstring(f.read()))
self.add_timestamps(item)
self.move_file(self.path, filename, success=True)
yield [item]
else:
self.move_file(self.path, filename, success=True)
except Exception as err:
logger.exception(err)
self.move_file(self.path, filename, success=False)
push_notification('ingest:update')
def _update(self, provider):
self.provider = provider
self.path = provider.get('config', {}).get('path', None)
if not self.path:
return []
for filename in get_sorted_files(self.path, sort_by=FileSortAttributes.created):
try:
filepath = os.path.join(self.path, filename)
if os.path.isfile(filepath):
stat = os.lstat(filepath)
last_updated = datetime.fromtimestamp(stat.st_mtime, tz=utc)
if self.is_latest_content(last_updated, provider.get('last_updated')):
with open(filepath, 'r') as f:
item = self.parser.parse_message(etree.fromstring(f.read()), provider)
self.move_file(self.path, filename, provider=provider, success=True)
yield [item]
else:
self.move_file(self.path, filename, provider=provider, success=True)
except etreeParserError as ex:
logger.exception("Ingest Type: AAP - File: {0} could not be processed".format(filename))
self.move_file(self.path, filename, provider=provider, success=False)
raise ParserError.nitfParserError(ex, provider)
except ParserError as ex:
self.move_file(self.path, filename, provider=provider, success=False)
except Exception as ex:
self.move_file(self.path, filename, provider=provider, success=False)
raise ProviderError.ingestError(ex, provider)
push_notification('ingest:update')
def _update(self, provider):
self.provider = provider
self.path = provider.get('config', {}).get('path', None)
if not self.path:
return
for filename in get_sorted_files(self.path,
sort_by=FileSortAttributes.created):
try:
if os.path.isfile(os.path.join(self.path, filename)):
filepath = os.path.join(self.path, filename)
stat = os.lstat(filepath)
last_updated = datetime.fromtimestamp(stat.st_mtime,
tz=utc)
if self.is_latest_content(last_updated,
provider.get('last_updated')):
with open(os.path.join(self.path, filename), 'r') as f:
item = self.parser.parse_message(
etree.fromstring(f.read()))
item['firstcreated'] \
= normalize_date(item.get('firstcreated'), self.tz)
item['versioncreated'] \
= normalize_date(item.get('versioncreated'), self.tz)
self.move_file(self.path, filename, success=True)
yield [item]
else:
self.move_file(self.path, filename, success=True)
except Exception as err:
logger.exception(err)
self.move_file(self.path, filename, success=False)
push_notification('ingest:update')
def _update(self, provider):
self.provider = provider
self.path = provider.get('config', {}).get('path', None)
if not self.path:
logger.info('No path')
return []
for filename in get_sorted_files(self.path, sort_by=FileSortAttributes.created):
try:
filepath = os.path.join(self.path, filename)
if os.path.isfile(filepath):
stat = os.lstat(filepath)
last_updated = datetime.fromtimestamp(stat.st_mtime, tz=utc)
if self.is_latest_content(last_updated, provider.get('last_updated')):
item = self.parser.parse_file(filepath, self)
self.move_file(self.path, filename, provider=provider, success=True)
yield [item]
else:
self.move_file(self.path, filename, provider=provider, success=True)
except ParserError.ZCZCParserError as ex:
logger.exception("Ingest Type: Teletype - File: {0} could not be processed".format(filename))
self.move_file(self.path, filename, provider=provider, success=False)
raise ParserError.ZCZCParserError(ex, provider)
except ParserError as ex:
self.move_file(self.path, filename, provider=provider, success=False)
except Exception as ex:
self.move_file(self.path, filename, provider=provider, success=False)
raise ProviderError.ingestError(ex, provider)
def _update(self, provider):
self.provider = provider
self.path = provider.get('config', {}).get('path', None)
if not self.path:
logger.info('No path')
return []
for filename in get_sorted_files(self.path, sort_by=FileSortAttributes.created):
try:
filepath = os.path.join(self.path, filename)
if os.path.isfile(filepath):
stat = os.lstat(filepath)
last_updated = datetime.fromtimestamp(stat.st_mtime, tz=utc)
if self.is_latest_content(last_updated, provider.get('last_updated')):
item = self.parser.parse_file(filepath, provider)
dpa_derive_dateline(item)
self.move_file(self.path, filename, provider=provider, success=True)
yield [item]
else:
self.move_file(self.path, filename, provider=provider, success=True)
except Exception as ex:
self.move_file(self.path, filename, provider=provider, success=False)
raise ParserError.parseFileError('DPA', filename, ex, provider)
def _update(self, provider, update):
self.provider = provider
self.path = provider.get('config', {}).get('path', None)
if not self.path:
logger.warn(
'File Feeding Service {} is configured without path. Please check the configuration'
.format(provider['name']))
return []
registered_parser = self.get_feed_parser(provider)
for filename in get_sorted_files(self.path,
sort_by=FileSortAttributes.created):
try:
last_updated = None
file_path = os.path.join(self.path, filename)
if os.path.isfile(file_path):
stat = os.lstat(file_path)
last_updated = datetime.fromtimestamp(stat.st_mtime,
tz=utc)
if self.is_latest_content(last_updated,
provider.get('last_updated')):
if isinstance(registered_parser, XMLFeedParser):
with open(file_path, 'rb') as f:
xml = etree.parse(f)
parser = self.get_feed_parser(
provider, xml.getroot())
item = parser.parse(xml.getroot(), provider)
else:
parser = self.get_feed_parser(provider, file_path)
item = parser.parse(file_path, provider)
self.after_extracting(item, provider)
if isinstance(item, list):
failed = yield item
else:
failed = yield [item]
self.move_file(self.path,
filename,
provider=provider,
success=not failed)
else:
self.move_file(self.path,
filename,
provider=provider,
success=True)
except Exception as ex:
if last_updated and self.is_old_content(last_updated):
self.move_file(self.path,
filename,
provider=provider,
success=False)
raise ParserError.parseFileError(
'{}-{}'.format(provider['name'], self.NAME), filename, ex,
provider)
push_notification('ingest:update')
def _update(self, provider, update):
# check if deprecated FILE_INGEST_OLD_CONTENT_MINUTES setting is still used
if "FILE_INGEST_OLD_CONTENT_MINUTES" in app.config:
deprecated_cont_min = app.config["FILE_INGEST_OLD_CONTENT_MINUTES"]
cont_min = app.config[OLD_CONTENT_MINUTES]
if deprecated_cont_min != cont_min:
logger.warning(
"'FILE_INGEST_OLD_CONTENT_MINUTES' is deprecated, please update settings.py to use {new_name!r}"
.format(new_name=OLD_CONTENT_MINUTES))
app.config[OLD_CONTENT_MINUTES] = deprecated_cont_min
self.provider = provider
self.path = provider.get('config', {}).get('path', None)
if not self.path:
logger.warn('File Feeding Service {} is configured without path. Please check the configuration'
.format(provider['name']))
return []
registered_parser = self.get_feed_parser(provider)
for filename in get_sorted_files(self.path, sort_by=FileSortAttributes.created):
try:
last_updated = None
file_path = os.path.join(self.path, filename)
if os.path.isfile(file_path):
last_updated = self.get_last_updated(file_path)
if self.is_latest_content(last_updated, provider.get('last_updated')):
if isinstance(registered_parser, XMLFeedParser):
with open(file_path, 'rb') as f:
xml = etree.parse(f)
parser = self.get_feed_parser(provider, xml.getroot())
item = parser.parse(xml.getroot(), provider)
else:
parser = self.get_feed_parser(provider, file_path)
item = parser.parse(file_path, provider)
self.after_extracting(item, provider)
if isinstance(item, list):
failed = yield item
else:
failed = yield [item]
self.move_file(self.path, filename, provider=provider, success=not failed)
else:
self.move_file(self.path, filename, provider=provider, success=False)
except Exception as ex:
if last_updated and self.is_old_content(last_updated):
self.move_file(self.path, filename, provider=provider, success=False)
raise ParserError.parseFileError('{}-{}'.format(provider['name'], self.NAME), filename, ex, provider)
push_notification('ingest:update')
def _update(self, provider):
self.provider = provider
self.path = provider.get("config", {}).get("path", None)
if not self.path:
logger.warn(
"File Feeding Service {} is configured without path. Please check the configuration".format(
provider["name"]
)
)
return []
registered_parser = self.get_feed_parser(provider)
for filename in get_sorted_files(self.path, sort_by=FileSortAttributes.created):
try:
file_path = os.path.join(self.path, filename)
if os.path.isfile(file_path):
stat = os.lstat(file_path)
last_updated = datetime.fromtimestamp(stat.st_mtime, tz=utc)
if self.is_latest_content(last_updated, provider.get("last_updated")):
if isinstance(registered_parser, XMLFeedParser):
with open(file_path, "rt") as f:
xml = ElementTree.parse(f)
parser = self.get_feed_parser(provider, xml.getroot())
item = parser.parse(xml, provider)
else:
parser = self.get_feed_parser(provider, file_path)
item = parser.parse(file_path, provider)
self.after_extracting(item, provider)
self.move_file(self.path, filename, provider=provider, success=True)
if isinstance(item, list):
yield item
else:
yield [item]
else:
self.move_file(self.path, filename, provider=provider, success=True)
except Exception as ex:
self.move_file(self.path, filename, provider=provider, success=False)
raise ParserError.parseFileError("{}-{}".format(provider["name"], self.NAME), filename, ex, provider)
push_notification("ingest:update")
def _update(self, provider):
self.provider = provider
self.path = provider.get('config', {}).get('path', None)
if not self.path:
logger.warn('File Feeding Service {} is configured without path. Please check the configuration'
.format(provider['name']))
return []
registered_parser = self.get_feed_parser(provider)
for filename in get_sorted_files(self.path, sort_by=FileSortAttributes.created):
try:
file_path = os.path.join(self.path, filename)
if os.path.isfile(file_path):
stat = os.lstat(file_path)
last_updated = datetime.fromtimestamp(stat.st_mtime, tz=utc)
if self.is_latest_content(last_updated, provider.get('last_updated')):
if isinstance(registered_parser, XMLFeedParser):
with open(file_path, 'r') as f:
xml_string = etree.fromstring(f.read())
parser = self.get_feed_parser(provider, xml_string)
item = parser.parse(xml_string, provider)
else:
parser = self.get_feed_parser(provider, file_path)
item = parser.parse(file_path, provider)
self.after_extracting(item, provider)
self.move_file(self.path, filename, provider=provider, success=True)
yield [item]
else:
self.move_file(self.path, filename, provider=provider, success=True)
except Exception as ex:
self.move_file(self.path, filename, provider=provider, success=False)
raise ParserError.parseFileError('{}-{}'.format(provider['name'], self.NAME), filename, ex, provider)
push_notification('ingest:update')
