def import_events(event_type, csv_dir, chunk_size):
r"""Import stats events from a directory of CSV files.
Available event types: "file-download", "record-view"
The following columns should always be present:
\b
- ipAddress
- userAgent
- url ("https://zenodo.org/record/1234/files/article.pdf")
- timestamp (1388506249)
- referrer ("Google", "example.com", etc)
"""
csv_files = glob.glob(csv_dir + '/*.csv')
with click.progressbar(csv_files, len(csv_files)) as csv_files_bar:
for csv_path in csv_files_bar:
with open(csv_path, 'r' if PY3 else 'rb') as fp:
reader = csv.DictReader(fp, delimiter=',')
events = filter(
None, map(EVENT_TYPE_BUILDERS[event_type], reader))
for event_chunk in chunkify(events, chunk_size):
current_stats.publish(event_type, event_chunk)
click.secho(
'Run the "invenio_stats.tasks.process_events" to index the events...',
fg='yellow')
def run(self, start_date=None, end_date=None, update_bookmark=True):
"""Run export job."""
if start_date is None:
bookmark = current_cache.get('piwik_export:bookmark')
start_date = dateutil_parse(bookmark) if bookmark else None
time_range = {}
if start_date is not None:
time_range['gte'] = start_date.replace(microsecond=0).isoformat()
if end_date is not None:
time_range['lte'] = end_date.replace(microsecond=0).isoformat()
events = Search(using=current_search_client,
index='events-stats-*').filter(
'range', timestamp=time_range).sort({
'timestamp': {
'order': 'asc'
}
}).params(preserve_order=True).scan()
url = current_app.config['ZENODO_STATS_PIWIK_EXPORTER'].get(
'url', None)
token_auth = current_app.config['ZENODO_STATS_PIWIK_EXPORTER'] \
.get('token_auth', None)
chunk_size = current_app.config['ZENODO_STATS_PIWIK_EXPORTER']\
.get('chunk_size', 0)
for event_chunk in chunkify(events, chunk_size):
query_strings = []
for event in event_chunk:
query_string = self._build_query_string(event)
query_strings.append(query_string)
payload = {'requests': query_strings, 'token_auth': token_auth}
res = requests.post(url, json=payload)
# Failure: not 200 or not "success"
content = res.json() if res.ok else None
if res.status_code == 200 and content.get('status') == 'success':
if content.get('invalid') != 0:
msg = 'Invalid events in Piwik export request.'
info = {
'begin_event_timestamp': event_chunk[0].timestamp,
'end_event_timestamp': event_chunk[-1].timestamp,
'invalid_events': content.get('invalid')
}
current_app.logger.warning(msg, extra=info)
elif update_bookmark is True:
current_cache.set('piwik_export:bookmark',
event_chunk[-1].timestamp,
timeout=-1)
else:
msg = 'Invalid events in Piwik export request.'
info = {
'begin_event_timestamp': event_chunk[0].timestamp,
'end_event_timestamp': event_chunk[-1].timestamp,
}
raise PiwikExportRequestError(msg, export_info=info)
def run(self, start_date=None, end_date=None, update_bookmark=True):
"""Run export job."""
if start_date is None:
bookmark = current_cache.get('piwik_export:bookmark')
if bookmark is None:
msg = 'Bookmark not found, and no start date specified.'
current_app.logger.warning(msg)
return
start_date = dateutil_parse(bookmark) if bookmark else None
time_range = {}
if start_date is not None:
time_range['gte'] = start_date.replace(microsecond=0).isoformat()
if end_date is not None:
time_range['lte'] = end_date.replace(microsecond=0).isoformat()
events = Search(using=current_search_client,
index=build_alias_name('events-stats-*')).filter(
'range', timestamp=time_range).sort({
'timestamp': {
'order': 'asc'
}
}).params(preserve_order=True).scan()
url = current_app.config['ZENODO_STATS_PIWIK_EXPORTER'].get(
'url', None)
token_auth = current_app.config['ZENODO_STATS_PIWIK_EXPORTER'] \
.get('token_auth', None)
chunk_size = current_app.config['ZENODO_STATS_PIWIK_EXPORTER']\
.get('chunk_size', 0)
for event_chunk in chunkify(events, chunk_size):
query_strings = []
for event in event_chunk:
if 'recid' not in event:
continue
try:
query_string = self._build_query_string(event)
query_strings.append(query_string)
except PIDDeletedError:
pass
# Check and bail if the bookmark has progressed, e.g. from another
# duplicate task or manual run of the exporter.
bookmark = current_cache.get('piwik_export:bookmark')
if event_chunk[-1].timestamp < bookmark:
return
payload = {'requests': query_strings, 'token_auth': token_auth}
res = requests.post(url, json=payload, timeout=60)
# Failure: not 200 or not "success"
content = res.json() if res.ok else None
if res.status_code == 200 and content.get('status') == 'success':
if content.get('invalid') != 0:
msg = 'Invalid events in Piwik export request.'
info = {
'begin_event_timestamp': event_chunk[0].timestamp,
'end_event_timestamp': event_chunk[-1].timestamp,
'invalid_events': content.get('invalid')
}
current_app.logger.warning(msg, extra=info)
elif update_bookmark is True:
current_cache.set('piwik_export:bookmark',
event_chunk[-1].timestamp,
timeout=-1)
else:
msg = 'Invalid events in Piwik export request.'
info = {
'begin_event_timestamp': event_chunk[0].timestamp,
'end_event_timestamp': event_chunk[-1].timestamp,
}
raise PiwikExportRequestError(msg, export_info=info)
def run(self, start_date=None, end_date=None, update_bookmark=True):
"""Run export job."""
if start_date is None:
bookmark = current_cache.get('piwik_export:bookmark')
if bookmark is None:
msg = 'Bookmark not found, and no start date specified.'
current_app.logger.warning(msg)
return
start_date = dateutil_parse(bookmark) if bookmark else None
time_range = {}
if start_date is not None:
time_range['gte'] = start_date.replace(microsecond=0).isoformat()
if end_date is not None:
time_range['lte'] = end_date.replace(microsecond=0).isoformat()
events = Search(
using=current_search_client,
index='events-stats-*'
).filter(
'range', timestamp=time_range
).sort(
{'timestamp': {'order': 'asc'}}
).params(preserve_order=True).scan()
url = current_app.config['ZENODO_STATS_PIWIK_EXPORTER'].get('url', None)
token_auth = current_app.config['ZENODO_STATS_PIWIK_EXPORTER'] \
.get('token_auth', None)
chunk_size = current_app.config['ZENODO_STATS_PIWIK_EXPORTER']\
.get('chunk_size', 0)
for event_chunk in chunkify(events, chunk_size):
query_strings = []
for event in event_chunk:
if 'recid' not in event:
continue
try:
query_string = self._build_query_string(event)
query_strings.append(query_string)
except PIDDeletedError:
pass
payload = {
'requests': query_strings,
'token_auth': token_auth
}
res = requests.post(url, json=payload)
# Failure: not 200 or not "success"
content = res.json() if res.ok else None
if res.status_code == 200 and content.get('status') == 'success':
if content.get('invalid') != 0:
msg = 'Invalid events in Piwik export request.'
info = {
'begin_event_timestamp': event_chunk[0].timestamp,
'end_event_timestamp': event_chunk[-1].timestamp,
'invalid_events': content.get('invalid')
}
current_app.logger.warning(msg, extra=info)
elif update_bookmark is True:
current_cache.set('piwik_export:bookmark',
event_chunk[-1].timestamp,
timeout=-1)
else:
msg = 'Invalid events in Piwik export request.'
info = {
'begin_event_timestamp': event_chunk[0].timestamp,
'end_event_timestamp': event_chunk[-1].timestamp,
}
raise PiwikExportRequestError(msg, export_info=info)