def parse(self, *args):
"""Parse a list of arguments.
Parse argument strings needed to run a backend command. The result
will be a `argparse.Namespace` object populated with the values
obtained after the validation of the parameters.
:param args: argument strings
:result: an object with the parsed values
"""
parsed_args = self.parser.parse_args(args)
if self._from_date:
parsed_args.from_date = str_to_datetime(parsed_args.from_date)
if self._to_date and parsed_args.to_date:
parsed_args.to_date = str_to_datetime(parsed_args.to_date)
if self._cache and parsed_args.fetch_cache and parsed_args.no_cache:
raise AttributeError(
"fetch-cache and no-cache arguments are not compatible")
# Set aliases
for alias, arg in self.aliases.items():
if (alias not in parsed_args) and (arg in parsed_args):
value = getattr(parsed_args, arg, None)
setattr(parsed_args, alias, value)
return parsed_args
def parse_answer_container(update_info):
"""Parse the answer info container of a given HTML question.
The method parses the information available in the answer information
container. The container can have up to 2 elements: the first one
contains the information related with the user who generated the question
and the date (if any). The second one contains the date of the updated,
and the user who updated it (if not the same who generated the question).
:param update_info: beautiful soup update_info container element
:returns: an object with the parsed information
"""
container_info = {}
created = update_info[0]
answered_at = created.abbr.attrs["title"]
# Convert date to UNIX timestamp
container_info['added_at'] = str(
str_to_datetime(answered_at).timestamp())
container_info['answered_by'] = AskbotParser.parse_user_info(
created)
try:
update_info[1]
except IndexError:
pass
else:
updated = update_info[1]
updated_at = updated.abbr.attrs["title"]
# Convert date to UNIX timestamp
container_info['updated_at'] = str(
str_to_datetime(updated_at).timestamp())
if AskbotParser.parse_user_info(updated):
container_info[
'updated_by'] = AskbotParser.parse_user_info(updated)
return container_info
def test_items_storing_archive(self):
"""Test whether items are stored in an archive"""
manager = ArchiveManager(self.test_path)
args = {
'origin': 'http://example.com/',
'category': 'mock_item',
'tag': 'test',
'subtype': 'mocksubtype',
'from-date': str_to_datetime('2015-01-01')
}
items = fetch(CommandBackend, args, manager=manager)
items = [item for item in items]
self.assertEqual(len(items), 5)
for x in range(5):
item = items[x]
expected_uuid = uuid('http://example.com/', str(x))
self.assertEqual(item['data']['item'], x)
self.assertEqual(item['origin'], 'http://example.com/')
self.assertEqual(item['uuid'], expected_uuid)
self.assertEqual(item['tag'], 'test')
filepaths = manager.search('http://example.com/', 'CommandBackend',
'mock_item', str_to_datetime('1970-01-01'))
self.assertEqual(len(filepaths), 1)
archive = Archive(filepaths[0])
self.assertEqual(archive._count_table_rows('archive'), 5)
def _validate_message(self, message):
"""Check if the given message has the mandatory fields"""
# This check is "case insensitive" because we're
# using 'CaseInsensitiveDict' from requests.structures
# module to store the contents of a message.
if self.MESSAGE_ID_FIELD not in message:
logger.warning("Field 'Message-ID' not found in message %s; ignoring",
message['unixfrom'])
return False
if not message[self.MESSAGE_ID_FIELD]:
logger.warning("Field 'Message-ID' is empty in message %s; ignoring",
message['unixfrom'])
return False
if self.DATE_FIELD not in message:
logger.warning("Field 'Date' not found in message %s; ignoring",
message['unixfrom'])
return False
if not message[self.DATE_FIELD]:
logger.warning("Field 'Date' is empty in message %s; ignoring",
message['unixfrom'])
return False
try:
str_to_datetime(message[self.DATE_FIELD])
except InvalidDateError:
logger.warning("Invalid date %s in message %s; ignoring",
message[self.DATE_FIELD], message['unixfrom'])
return False
return True
def test_archived_after(self):
"""Test if only those items archived after a date are returned"""
manager = ArchiveManager(self.test_path)
args = {
'origin': 'http://example.com/',
'category': 'mock_item',
'tag': 'test',
'subtype': 'mocksubtype',
'from-date': str_to_datetime('2015-01-01')
}
items = fetch(CommandBackend, args, manager=manager)
items = [item for item in items]
self.assertEqual(len(items), 5)
archived_dt = datetime_utcnow()
items = fetch(CommandBackend, args, manager=manager)
items = [item for item in items]
self.assertEqual(len(items), 5)
# Fetch items from the archive
items = fetch_from_archive(CommandBackend, args, manager, 'mock_item',
str_to_datetime('1970-01-01'))
items = [item for item in items]
self.assertEqual(len(items), 10)
# Fetch items archived after the given date
items = fetch_from_archive(CommandBackend, args, manager, 'mock_item',
archived_dt)
items = [item for item in items]
self.assertEqual(len(items), 5)
def _run(self):
issues_iter = self.backend.fetch()
# Import all issues
for issue_d in issues_iter:
issue = issue_d['data']
closed_at = issue['closed_at']
if closed_at:
closed_at = str_to_datetime(issue['closed_at'])
bug, created = self.object.bugs.get_or_create(
bug_id=str(issue['number']),
defaults={
'close_date': closed_at,
'severity': None
})
logger.info("%s bug [%s]", "Imported" if created else "Found", bug)
if created:
bug_create_time = str_to_datetime(issue['created_at'])
comment = bug.comments.create(comment_id='VIRTUAL-1',
author=self.get_user(
issue['user_data']),
timestamp=bug_create_time)
self.record_timestamp(bug_create_time)
logger.info("Imported bug body as [%s]", comment)
# TODO: not supported yet
comments = []
# Import comments
for comment in comments:
issue_number = os.path.basename(comment['issue_url'])
bug = self.object.bugs.get(bug_id=issue_number)
extrafields = {
'author': self.get_user(comment['user']['login']),
'timestamp': str_to_datetime(comment['created_at'])
}
comment, created = bug.comments.get_or_create(
comment_id=comment['id'], defaults=extrafields)
logger.info("%s comment [%s]", "Imported" if created else "Found",
comment)
if not created:
for key, value in extrafields.items():
oldvalue = getattr(comment, key)
if oldvalue != value:
logger.warning(
"Updating field [%s] for comment [%s] "
"(%s -> %s)", key, comment, oldvalue, value)
setattr(comment, key, value)
self.record_timestamp(comment.timestamp)
else:
self.record_timestamp(comment.timestamp)
def test_ignore_corrupted_archive(self):
"""Check if a corrupted archive is ignored while fetching from archive"""
def delete_rows(db, table_name):
conn = sqlite3.connect(db)
cursor = conn.cursor()
cursor.execute("DELETE FROM " + table_name)
cursor.close()
conn.commit()
manager = ArchiveManager(self.test_path)
args = {
'origin': 'http://example.com/',
'category': 'mock_item',
'tag': 'test',
'subtype': 'mocksubtype',
'from-date': str_to_datetime('2015-01-01')
}
# First, fetch the items twice to check if several archive
# are used
items = fetch(CommandBackend, args, manager=manager)
items = [item for item in items]
self.assertEqual(len(items), 5)
items = fetch(CommandBackend, args, manager=manager)
items = [item for item in items]
self.assertEqual(len(items), 5)
# Find archive names to delete the rows of one of them to make it
# corrupted
filepaths = manager.search('http://example.com/', 'CommandBackend',
'mock_item', str_to_datetime('1970-01-01'))
self.assertEqual(len(filepaths), 2)
to_remove = filepaths[0]
delete_rows(to_remove, 'archive')
# Fetch items from the archive
items = fetch_from_archive(CommandBackend, args, manager,
'mock_item', str_to_datetime('1970-01-01'))
items = [item for item in items]
self.assertEqual(len(items), 5)
for x in range(5):
item = items[x]
expected_uuid = uuid('http://example.com/', str(x))
self.assertEqual(item['data']['item'], x)
self.assertEqual(item['data']['archive'], True)
self.assertEqual(item['origin'], 'http://example.com/')
self.assertEqual(item['uuid'], expected_uuid)
self.assertEqual(item['tag'], 'test')
def test_get_issues_empty(self):
"""Test get when the issue is empty API call"""
from_date = str_to_datetime('2015-01-01')
body = '{"total": 0, "maxResults": 0, "startAt": 0}'
httpretty.register_uri(httpretty.GET,
JIRA_SEARCH_URL,
body=body, status=200)
client = JiraClient(url='http://example.com', project='perceval',
user='******', password='******',
verify=False, cert=None, max_issues=1)
pages = [page for page in client.get_issues(from_date)]
expected_req = {
'expand': ['renderedFields,transitions,operations,changelog'],
'jql': ['project = perceval AND updated > 1420070400000 order by updated asc'],
'maxResults': ['1'],
'startAt': ['0']
}
self.assertEqual(len(pages), 1)
self.assertEqual(pages[0], body)
self.assertDictEqual(httpretty.last_request().querystring, expected_req)
def fetch(self, from_date=DEFAULT_DATETIME):
"""Fetch the messages from the Supybot IRC logger.
The method parsers and returns the messages saved on the
IRC log files and stored by Supybot in `dirpath`.
:param from_date: obtain messages since this date
:returns: a generator of messages
"""
logger.info("Fetching messages of '%s' from %s",
self.uri, str(from_date))
from_date = datetime_to_utc(from_date)
nmessages = 0
archives = self.__retrieve_archives(from_date)
for archive in archives:
logger.debug("Parsing supybot archive %s", archive)
for message in self.parse_supybot_log(archive):
dt = str_to_datetime(message['timestamp'])
if dt < from_date:
logger.debug("Message %s sent before %s; skipped",
str(dt), str(from_date))
continue
yield message
nmessages += 1
logger.info("Fetch process completed: %s messages fetched",
nmessages)
def __fetch_crates(self, from_date):
"""Fetch crates"""
from_date = datetime_to_utc(from_date)
crates_groups = self.client.crates()
for raw_crates in crates_groups:
crates = json.loads(raw_crates)
for crate_container in crates['crates']:
if str_to_datetime(crate_container['updated_at']) < from_date:
continue
crate_id = crate_container['id']
crate = self.__fetch_crate_data(crate_id)
crate['owner_team_data'] = self.__fetch_crate_owner_team(
crate_id)
crate['owner_user_data'] = self.__fetch_crate_owner_user(
crate_id)
crate[
'version_downloads_data'] = self.__fetch_crate_version_downloads(
crate_id)
crate['versions_data'] = self.__fetch_crate_versions(crate_id)
yield crate
def _load_metadata(self):
"""Load metadata from the archive file"""
logger.debug("Loading metadata infomation of archive %s",
self.archive_path)
cursor = self._db.cursor()
select_stmt = "SELECT origin, backend_name, backend_version, " \
"category, backend_params, created_on " \
"FROM " + self.METADATA_TABLE + " " \
"LIMIT 1"
cursor.execute(select_stmt)
row = cursor.fetchone()
cursor.close()
if row:
self.origin = row[0]
self.backend_name = row[1]
self.backend_version = row[2]
self.category = row[3]
self.backend_params = pickle.loads(row[4])
self.created_on = str_to_datetime(row[5])
else:
logger.debug("Metadata of archive %s was empty", self.archive_path)
logger.debug("Metadata of archive %s loaded", self.archive_path)
def __parse_topics_page(self, raw_json):
"""Parse a topics page stream.
The result of parsing process is a generator of tuples. Each
tuple contains de identifier of the topic, the last date
when it was updated and whether is pinned or not.
:param raw_json: JSON stream to parse
:returns: a generator of parsed bugs
"""
topics_page = json.loads(raw_json)
topics_ids = []
for topic in topics_page['topic_list']['topics']:
topic_id = topic['id']
if topic['last_posted_at'] is None:
logger.warning(
"Topic %s with last_posted_at null. Ignoring it.",
topic['title'])
continue
updated_at = str_to_datetime(topic['last_posted_at'])
pinned = topic['pinned']
topics_ids.append((topic_id, updated_at, pinned))
return topics_ids
def __parse_archive_args(self, archive_args):
"""Parse the archive arguments of a task"""
if not archive_args:
return {}
if self.archive_path:
archive_args['archive_path'] = self.archive_path
else:
archive_args['archive_path'] = os.path.expanduser(ARCHIVES_DEFAULT_PATH)
if 'fetch_from_archive' not in archive_args:
raise ValueError("archive_args.fetch_from_archive not defined")
if archive_args['fetch_from_archive'] and 'archived_after' not in archive_args:
raise ValueError("archive_args.archived_after not defined")
for arg in archive_args.keys():
if arg == 'archive_path':
continue
elif arg == 'fetch_from_archive':
if type(archive_args['fetch_from_archive']) is not bool:
raise ValueError("archive_args.fetch_from_archive not boolean")
elif arg == 'archived_after':
if archive_args['fetch_from_archive']:
try:
archive_args['archived_after'] = str_to_datetime(archive_args['archived_after'])
except InvalidDateError:
raise ValueError("archive_args.archived_after datetime format not valid")
else:
archive_args['archived_after'] = None
else:
raise ValueError("%s not accepted in archive_args" % arg)
return archive_args
def test_get_pages_from_allrevisions_from_date(self):
HTTPServer.routes()
body = read_file('data/mediawiki/mediawiki_pages_allrevisions.json')
client = MediaWikiClient(MEDIAWIKI_SERVER_URL)
namespaces = ['0']
str_date = '2016-01-01 00:00'
dt = str_to_datetime(str_date)
from_date = datetime_to_utc(dt)
response = client.get_pages_from_allrevisions(namespaces, from_date)
req = HTTPServer.requests_http[-1]
self.assertEqual(response, body)
self.assertEqual(req.method, 'GET')
self.assertRegex(req.path, '/api.php')
# Check request params
expected = {
'action': ['query'],
'list': ['allrevisions'],
'arvnamespace': ['0'],
'arvdir': ['newer'],
'arvlimit': ['max'],
'format': ['json'],
'arvprop': ['ids'],
'arvstart': ['2016-01-01T00:00:00Z']
}
self.assertDictEqual(req.querystring, expected)
from_date = datetime.datetime(2016, 1, 1, 0, 0, 0)
with self.assertRaises(ValueError):
_ = client.get_pages_from_allrevisions(namespaces, from_date)
def _import_comment(self, bug, comment_id, author, timestamp):
"""
Import comment into `bug'.
@arg bug btinfo.models.Bug object to import comment for
@arg comment_id String representing comment ID
@arg author Jira user
@arg timestamp String timestamp in iso8601 form
@returns True if comment was created, otherwise false
"""
timestamp = str_to_datetime(timestamp)
name = author['displayName']
email = author.get('emailAddress', None)
author = get_participant(name, email)
_, created = bug.comments.get_or_create(comment_id=comment_id,
defaults={
'author': author,
'timestamp': timestamp
})
if created:
self.record_timestamp(timestamp)
return created
def latest_date(self):
"""Get date of most recent item available in ElasticSearch.
:return: latest date based on `CeresBase._sort_on` field,
None if no values found for that field.
:raises NotFoundError: index not found in ElasticSearch
"""
latest_date = None
search = Search(using=self._es_conn, index=self._es_index)
# from:to parameters (=> from: 0, size: 0)
search = search[0:0]
search = search.aggs.metric('max_date',
'max',
field=self._sort_on_field)
try:
response = search.execute()
aggs = response.to_dict()['aggregations']
if aggs['max_date']['value'] is None:
logger.debug("No data for " + self._sort_on_field +
" field found in " + self._es_index + " index")
else:
# Incremental case: retrieve items from last item in ES write index
max_date = aggs['max_date']['value_as_string']
latest_date = datetime.str_to_datetime(max_date).isoformat()
except NotFoundError as nfe:
raise nfe
return latest_date
def add(self):
"""Add tasks"""
payload = cherrypy.request.json
logger.debug("Reading tasks...")
for task_data in payload['tasks']:
try:
category = task_data['category']
backend_args = task_data['backend_args']
archive_args = task_data['archive']
sched_args = task_data['scheduler']
except KeyError as ex:
logger.error("Task badly formed")
raise ex
from_date = backend_args.get('from_date', None)
if from_date:
backend_args['from_date'] = str_to_datetime(from_date)
super().add_task(task_data['task_id'],
task_data['backend'],
category,
backend_args,
archive_args=archive_args,
sched_args=sched_args)
logger.debug("Done. Ready to work!")
return "Tasks added"
def fetch_items(self, category, **kwargs):
"""Fetch the messages
:param category: the category of items to fetch
:param kwargs: backend arguments
:returns: a generator of items
"""
from_date = kwargs['from_date']
logger.info("Fetching messages of '%s' from %s", self.uri,
str(from_date))
nmessages = 0
archives = self.__retrieve_archives(from_date)
for archive in archives:
logger.debug("Parsing supybot archive %s", archive)
for message in self.parse_supybot_log(archive):
dt = str_to_datetime(message['timestamp'])
if dt < from_date:
logger.debug("Message %s sent before %s; skipped", str(dt),
str(from_date))
continue
yield message
nmessages += 1
logger.info("Fetch process completed: %s messages fetched", nmessages)
def test_setup_cmd_parser(self):
"""Test if it parser object is correctly initialized"""
parser = GraalCommand.setup_cmd_parser()
self.assertIsInstance(parser, BackendCommandArgumentParser)
args = [
'http://example.com/', '--git-path', '/tmp/gitpath', '--tag',
'test'
]
parsed_args = parser.parse(*args)
self.assertEqual(parsed_args.uri, 'http://example.com/')
self.assertEqual(parsed_args.git_path, '/tmp/gitpath')
self.assertEqual(parsed_args.tag, 'test')
self.assertEqual(parsed_args.from_date, DEFAULT_DATETIME)
self.assertEqual(parsed_args.to_date, None)
self.assertEqual(parsed_args.branches, None)
self.assertFalse(parsed_args.latest_items)
self.assertEqual(parsed_args.worktreepath, DEFAULT_WORKTREE_PATH)
self.assertEqual(parsed_args.in_paths, None)
self.assertEqual(parsed_args.out_paths, None)
self.assertEqual(parsed_args.entrypoint, None)
self.assertFalse(parsed_args.details)
args = [
'http://example.com/', '--git-path', '/tmp/gitpath', '--tag',
'test', '--from-date', '1975-01-01', '--to-date', '2099-01-01',
'--branches', 'master', 'testing', '--latest-items',
'--worktree-path', '/tmp/custom-worktrees/', '--in-paths', '*.py',
'*.java', '--out-paths', '*.c', '--entrypoint', 'module',
'--details'
]
parsed_args = parser.parse(*args)
self.assertEqual(parsed_args.uri, 'http://example.com/')
self.assertEqual(parsed_args.git_path, '/tmp/gitpath')
self.assertEqual(parsed_args.tag, 'test')
self.assertEqual(parsed_args.from_date, str_to_datetime('1975-01-01'))
self.assertEqual(parsed_args.to_date, str_to_datetime('2099-01-01'))
self.assertEqual(parsed_args.branches, ['master', 'testing'])
self.assertTrue(parsed_args.latest_items)
self.assertEqual(parsed_args.worktreepath, '/tmp/custom-worktrees/')
self.assertEqual(parsed_args.in_paths, ['*.py', '*.java'])
self.assertEqual(parsed_args.out_paths, ['*.c'])
self.assertEqual(parsed_args.entrypoint, 'module')
self.assertTrue(parsed_args.details)
def fetch_items(self, category, **kwargs):
"""Fetch the tweets
:param category: the category of items to fetch
:param kwargs: backend arguments
:returns: a generator of items
"""
since_id = kwargs['since_id']
max_id = kwargs['max_id']
geocode = kwargs['geocode']
lang = kwargs['lang']
entities = kwargs['include_entities']
tweets_type = kwargs['result_type']
logger.info("Fetching tweets %s from %s to %s", self.query,
str(since_id),
str(max_id) if max_id else '--')
tweets_ids = []
min_date = None
max_date = None
group_tweets = self.client.tweets(self.query,
since_id=since_id,
max_id=max_id,
geocode=geocode,
lang=lang,
include_entities=entities,
result_type=tweets_type)
for tweets in group_tweets:
for i in range(len(tweets)):
tweet = tweets[i]
tweets_ids.append(tweet['id'])
if tweets[-1] == tweet:
min_date = str_to_datetime(tweets[-1]['created_at'])
if tweets[0] == tweet and not max_date:
max_date = str_to_datetime(tweets[0]['created_at'])
yield tweet
logger.info(
"Fetch process completed: %s (unique %s) tweets fetched, from %s to %s",
len(tweets_ids), len(list(set(tweets_ids))), min_date, max_date)
def metadata_id(item):
"""Extracts the identifier from an item depending on its type."""
if Crates.metadata_category(item) == CRATES_CATEGORY:
return str(item['id'])
else:
ts = item['fetched_on']
ts = str_to_datetime(ts)
return str(ts.timestamp())
def __get_max_date(self, reviews):
""""Get the max date in unixtime format from reviews."""
max_ts = 0
for review in reviews:
ts = str_to_datetime(review['timestamp'])
ts = datetime_to_utc(ts)
if ts.timestamp() > max_ts:
max_ts = ts.timestamp()
return max_ts
def test_get_issues(self):
"""Test get issues API call"""
from_date = str_to_datetime('2015-01-01')
requests = []
bodies_json = [read_file('data/jira/jira_issues_page_1.json'),
read_file('data/jira/jira_issues_page_2.json')]
bodies = bodies_json[:]
bodies = list(bodies_json)
def request_callback(method, uri, headers):
body = bodies.pop(0)
requests.append(httpretty.last_request())
return (200, headers, body)
httpretty.register_uri(httpretty.GET,
JIRA_SEARCH_URL,
responses=[httpretty.Response(body=request_callback)
for _ in range(2)])
client = JiraClient(url='http://example.com', project='perceval',
user='******', password='******',
verify=False, cert=None, max_issues=2)
pages = [page for page in client.get_issues(from_date)]
expected_req = [
{
'expand': ['renderedFields,transitions,operations,changelog'],
'jql': ['project = perceval AND updated > 1420070400000 order by updated asc'],
'maxResults': ['2'],
'startAt': ['0']
},
{
'expand': ['renderedFields,transitions,operations,changelog'],
'jql': ['project = perceval AND updated > 1420070400000 order by updated asc'],
'maxResults': ['2'],
'startAt': ['2']
}
]
self.assertEqual(len(pages), 2)
self.assertEqual(requests[0].method, 'GET')
self.assertRegex(requests[0].path, '/rest/api/2/search')
self.assertDictEqual(requests[0].querystring, expected_req[0])
self.assertEqual(requests[1].method, 'GET')
self.assertRegex(requests[1].path, '/rest/api/2/search')
self.assertDictEqual(requests[1].querystring, expected_req[1])
self.assertEqual(pages[0], bodies_json[0])
self.assertEqual(pages[1], bodies_json[1])
def __fix_field_date(self, item, attribute):
"""Fix possible errors in the field date"""
field_date = str_to_datetime(item[attribute])
try:
_ = int(field_date.strftime("%z")[0:3])
except ValueError:
logger.warning("%s in commit %s has a wrong format", attribute,
item['commit'])
item[attribute] = field_date.replace(tzinfo=None).isoformat()
def _run(self):
issues = self.backend.fetch()
for issue in issues:
data = issue['data']
severity = self.translate_severity(data['priority']['name'])
bug, created = self.object.bugs.get_or_create(
bug_id=data['id'], defaults={'severity': severity})
logger.info("%s bug [%s]", "Created" if created else "Found", bug)
if not created:
bug.severity = severity
bug.save()
else:
logger.info("Saving initial comment for [%s]", bug)
bug.comments.create(comment_id='VIRTUAL-1',
author=self.getuser(data['author_data']),
timestamp=str_to_datetime(
data['created_on']))
last_closed_time = None
for journal in data['journals']:
journal_time = str_to_datetime(journal['created_on'])
comment, created = bug.comments.get_or_create(
comment_id=journal['id'],
author=self.getuser(journal['user_data']),
timestamp=journal_time)
logger.info("%s comment [%s]",
"Created" if created else "Found", comment)
if self.journal_is_closing_entry(journal):
last_closed_time = journal_time
if last_closed_time is not None:
bug.close_date = last_closed_time
bug.save()
def metadata_updated_on(item):
"""Extracts the update time from a Kitsune item.
The timestamp is extracted from 'timestamp' field.
This date is a UNIX timestamp but needs to be converted to
a float value.
:param item: item generated by the backend
:returns: a UNIX timestamp
"""
return float(str_to_datetime(item['updated']).timestamp())
def test_archive(self):
"""Test whether a set of items is fetched from the archive"""
manager = ArchiveManager(self.test_path)
args = {
'origin': 'http://example.com/',
'category': 'mock_item',
'tag': 'test',
'subtype': 'mocksubtype',
'from-date': str_to_datetime('2015-01-01')
}
# First, fetch the items twice to check if several archive
# are used
items = fetch(CommandBackend, args, manager=manager)
items = [item for item in items]
self.assertEqual(len(items), 5)
items = fetch(CommandBackend, args, manager=manager)
items = [item for item in items]
self.assertEqual(len(items), 5)
# Fetch items from the archive
items = fetch_from_archive(CommandBackend, args, manager, 'mock_item',
str_to_datetime('1970-01-01'))
items = [item for item in items]
self.assertEqual(len(items), 10)
for x in range(2):
for y in range(5):
item = items[y + (x * 5)]
expected_uuid = uuid('http://example.com/', str(y))
self.assertEqual(item['data']['item'], y)
self.assertEqual(item['data']['archive'], True)
self.assertEqual(item['origin'], 'http://example.com/')
self.assertEqual(item['uuid'], expected_uuid)
self.assertEqual(item['tag'], 'test')
def __get_next_event(self, event_fields):
# Fill the empty event with all fields as None
event = {key: None for key in event_fields.values()}
event['updated'] = DEFAULT_DATETIME.isoformat()
last_col = 0
while self.ncell < len(self.cells):
# Get all cols (cells) for the event (row)
cell = self.cells[self.ncell]
ncol = int(cell['gs$cell']['col'])
if ncol <= last_col:
# new event (row) detected: new cell column lower than last
break
event[event_fields[ncol]] = cell['content']['$t']
# Add an extra column with the update datetime
cell_update = str_to_datetime(cell['updated']['$t'])
if cell_update > str_to_datetime(event['updated']):
event['updated'] = cell['updated']['$t']
last_col = ncol
self.ncell += 1
return event
def metadata_updated_on(item):
"""Extracts the update time from a MozillaClub item.
The timestamp is extracted from 'updated' field.
This date is in ISO format and it needs to be converted to
a float value.
:param item: item generated by the backend
:returns: a UNIX timestamp
"""
date = str_to_datetime(item['updated'])
return float(date.timestamp())
def test_remove_archive_on_error(self):
"""Test whether an archive is removed when an unhandled exception occurs"""
manager = ArchiveManager(self.test_path)
args = {
'origin': 'http://example.com/',
'category': 'mock_item',
'tag': 'test',
'subtype': 'mocksubtype',
'from-date': str_to_datetime('2015-01-01')
}
items = fetch(ErrorCommandBackend, args, manager=manager)
with self.assertRaises(BackendError):
_ = [item for item in items]
filepaths = manager.search('http://example.com/', 'ErrorCommandBackend',
'mock_item', str_to_datetime('1970-01-01'))
self.assertEqual(len(filepaths), 0)
def test_no_archived_items(self):
"""Test when no archived items are available"""
manager = ArchiveManager(self.test_path)
args = {
'origin': 'http://example.com/',
'category': 'mock_item',
'tag': 'test',
'subtype': 'mocksubtype',
'from-date': str_to_datetime('2015-01-01')
}
items = fetch(CommandBackend, args, manager=manager)
items = [item for item in items]
self.assertEqual(len(items), 5)
# There aren't items for this category
items = fetch_from_archive(CommandBackend, args, manager, 'alt_item',
str_to_datetime('1970-01-01'))
items = [item for item in items]
self.assertEqual(len(items), 0)
