def test_update_timestamp_without_last_updated(self):
result = timezone.datetime(2009, 1, 1).replace(tzinfo=timezone.utc)
with mock.patch.object(timezone, 'datetime',
mock.Mock(wraps=timezone.datetime)) as mock_:
mock_.now.return_value = result
TimeStampPlugin.get_last_updated('TestPluginUpdate')
TimeStampPlugin.update_timestamp('TestPluginUpdate')
timestamp = TimeStampPlugin.get_last_updated('TestPluginUpdate')
self.assertEquals(result, timestamp)
def test_update_timestamp_without_last_updated(self):
result = timezone.datetime(2009, 1, 1).replace(tzinfo=timezone.utc)
with mock.patch.object(timezone, 'datetime',
mock.Mock(wraps=timezone.datetime)) as mock_:
mock_.now.return_value = result
TimeStampPlugin.get_last_updated('TestPluginUpdate')
TimeStampPlugin.update_timestamp('TestPluginUpdate')
timestamp = TimeStampPlugin.get_last_updated('TestPluginUpdate')
self.assertEquals(result, timestamp)
def fetch_users(self):
url = '/api/v1/people'
timestamp = TimeStampPlugin.get_last_updated('NoosferoUser')
can_updated_timestamp = True
page = 1
while True:
json_data = self.get_json_data(url, page, timestamp=timestamp,
order="updated_at DESC")
if not len(json_data) or not len(json_data.get('people', [])):
break
json_data = json_data['people']
if can_updated_timestamp:
self.save_last_update(json_data[0]['updated_at'],
"NoosferoUser")
can_updated_timestamp = False
for element in json_data:
user = NoosferoUser()
user.id = element['id']
user.name = element['name']
user.username = element['identifier']
try:
user.save()
except:
continue
page += 1
def fetch_software_communities(self):
url = '/api/v1/software_communities'
timestamp = TimeStampPlugin.get_last_updated(
'NoosferoSoftwareCommunity')
can_updated_timestamp = True
page = 1
while True:
json_data = self.get_json_data(url, page, timestamp=timestamp,
order="updated_at DESC")
if not len(json_data) or not len(json_data.get('software_infos',
[])):
break
json_data = json_data['software_infos']
if can_updated_timestamp:
self.save_last_update(json_data[0]['updated_at'],
'NoosferoSoftwareCommunity')
can_updated_timestamp = False
for element in json_data:
software_community = NoosferoSoftwareCommunity()
self.fill_object_data(element, software_community)
try:
software_community.save()
except:
continue
page += 1
def fetch_users(self):
url = '/api/v1/people'
timestamp = TimeStampPlugin.get_last_updated('NoosferoUser')
can_updated_timestamp = True
page = 1
while True:
json_data = self.get_json_data(url,
page,
timestamp=timestamp,
order="updated_at DESC")
if not len(json_data) or not len(json_data.get('people', [])):
break
json_data = json_data['people']
if can_updated_timestamp:
self.save_last_update(json_data[0]['updated_at'],
"NoosferoUser")
can_updated_timestamp = False
for element in json_data:
user = NoosferoUser()
user.id = element['id']
user.name = element['name']
user.username = element['identifier']
try:
user.save()
except:
continue
page += 1
def fetch_software_communities(self):
url = '/api/v1/software_communities'
timestamp = TimeStampPlugin.get_last_updated(
'NoosferoSoftwareCommunity')
can_updated_timestamp = True
page = 1
while True:
json_data = self.get_json_data(url,
page,
timestamp=timestamp,
order="updated_at DESC")
if not len(json_data) or not len(
json_data.get('software_infos', [])):
break
json_data = json_data['software_infos']
if can_updated_timestamp:
self.save_last_update(json_data[0]['updated_at'],
'NoosferoSoftwareCommunity')
can_updated_timestamp = False
for element in json_data:
software_community = NoosferoSoftwareCommunity()
self.fill_object_data(element, software_community)
try:
software_community.save()
except:
continue
page += 1
def fetch_communities(self):
url = '/api/v1/communities'
timestamp = TimeStampPlugin.get_last_updated('NoosferoCommunity')
page = 1
can_updated_timestamp = True
while True:
json_data = self.get_json_data(url,
page,
timestamp=timestamp,
order="updated_at DESC")
if not len(json_data) or not len(json_data.get('communities', [])):
break
json_data = json_data['communities']
if can_updated_timestamp:
self.save_last_update(json_data[0]['updated_at'],
'NoosferoCommunity')
can_updated_timestamp = False
for element in json_data:
community = NoosferoCommunity()
self.fill_object_data(element, community)
if element['image']:
community.thumb_url = element['image']['thumb_url']
try:
community.save()
except:
continue
if 'categories' in element:
self.fetch_community_categories(community,
element["categories"])
if 'admins' in element:
self.fetch_software_admins(community, element["admins"])
page += 1
def fetch_articles(self):
url = '/api/v1/articles'
timestamp = TimeStampPlugin.get_last_updated('NoosferoArticle')
can_updated_timestamp = True
page = 1
while True:
json_data = self.get_json_data(url,
page,
timestamp=timestamp,
order="updated_at DESC",
show_comments="true")
if not len(json_data) or not len(json_data.get('articles', [])):
break
json_data = json_data['articles']
if can_updated_timestamp:
self.save_last_update(json_data[0]['updated_at'],
"NoosferoArticle")
can_updated_timestamp = False
for element in json_data:
article = NoosferoArticle()
self.fill_object_data(element, article)
try:
article.save()
except:
continue
self.import_comments(element["comments"], element["id"])
for category_json in element["categories"]:
category = NoosferoCategory.objects.get_or_create(
id=category_json["id"], name=category_json["name"])[0]
article.categories.add(category.id)
page += 1
def fetch_communities(self):
url = '/api/v1/communities'
timestamp = TimeStampPlugin.get_last_updated('NoosferoCommunity')
page = 1
can_updated_timestamp = True
while True:
json_data = self.get_json_data(url, page, timestamp=timestamp,
order="updated_at DESC")
if not len(json_data) or not len(json_data.get('communities', [])):
break
json_data = json_data['communities']
if can_updated_timestamp:
self.save_last_update(json_data[0]['updated_at'],
'NoosferoCommunity')
can_updated_timestamp = False
for element in json_data:
community = NoosferoCommunity()
self.fill_object_data(element, community)
if element['image']:
community.thumb_url = element['image']['thumb_url']
try:
community.save()
except:
continue
if 'categories' in element:
self.fetch_community_categories(community,
element["categories"])
if 'admins' in element:
self.fetch_software_admins(community, element["admins"])
page += 1
def fetch_articles(self):
url = '/api/v1/articles'
timestamp = TimeStampPlugin.get_last_updated('NoosferoArticle')
can_updated_timestamp = True
page = 1
while True:
json_data = self.get_json_data(url, page, timestamp=timestamp,
order="updated_at DESC",
show_comments="true")
if not len(json_data) or not len(json_data.get('articles', [])):
break
json_data = json_data['articles']
if can_updated_timestamp:
self.save_last_update(json_data[0]['updated_at'],
"NoosferoArticle")
can_updated_timestamp = False
for element in json_data:
article = NoosferoArticle()
self.fill_object_data(element, article)
try:
article.save()
except:
continue
self.import_comments(element["comments"], element["id"])
for category_json in element["categories"]:
category = NoosferoCategory.objects.get_or_create(
id=category_json["id"], name=category_json["name"])[0]
article.categories.add(category.id)
page += 1
def create_sample_timestamp(self, class_name, date):
TimeStampPlugin.get_last_updated(class_name)
TimeStampPlugin.update_timestamp(class_name, last_updated=date)
def test_first_get_last_update(self):
timestamp = TimeStampPlugin.get_last_updated('Test')
self.assertEqual(timezone.datetime.min, timestamp)
def test_first_get_last_update(self):
timestamp = TimeStampPlugin.get_last_updated('Test')
self.assertEqual(timezone.datetime.min, timestamp)
def save_last_update(self, last_updated, class_name):
TimeStampPlugin.update_timestamp(class_name, last_updated=last_updated)
def create_sample_timestamp(self, class_name, date):
TimeStampPlugin.get_last_updated(class_name)
TimeStampPlugin.update_timestamp(class_name, last_updated=date)
def save_last_update(self, last_updated, class_name):
TimeStampPlugin.update_timestamp(class_name, last_updated=last_updated)
def test_update_timestamp_with_last_updated(self):
date = '2015/12/23 00:00:00'
self.create_sample_timestamp('TestPluginUpdate', date)
timestamp = TimeStampPlugin.get_last_updated('TestPluginUpdate')
self.assertEquals(self.create_timestamp_object(date), timestamp)
def test_update_timestamp_with_last_updated(self):
date = '2015/12/23 00:00:00'
self.create_sample_timestamp('TestPluginUpdate', date)
timestamp = TimeStampPlugin.get_last_updated('TestPluginUpdate')
self.assertEquals(self.create_timestamp_object(date), timestamp)