def buzzfeed_fakenews_resource(buzzfeed_dataset, buzzfeed_editor, mocker):
from mcod.resources.models import Resource
_name = 'buzzfeed-2018-fake-news-1000-lines.csv'
copyfile(os.path.join(settings.TEST_DATA_PATH, _name),
os.path.join(settings.RESOURCES_MEDIA_ROOT, _name))
res = Resource(
title='Analysis of fake news sites and viral posts',
description=
'Over the past four years, BuzzFeed News has maintained lists of sites that '
'publish completely fabricated stories. As we encounter new ones and debunk '
'their content, we add them to the list.',
file=_name,
link=f'http://falconframework.org/media/resources/{_name}',
format='csv',
openness_score=3,
views_count=10,
downloads_count=20,
dataset=buzzfeed_dataset,
created_by=buzzfeed_editor,
modified_by=buzzfeed_editor,
data_date=datetime.today())
res.tracker.saved_data['link'] = res.link
res.save()
return res
def other_remote_file_resource(buzzfeed_dataset, buzzfeed_editor, mocker, httpserver):
from mcod.resources.models import Resource
simple_csv_path = os.path.join(settings.TEST_SAMPLES_PATH, 'simple.csv')
httpserver.serve_content(
content=open(simple_csv_path).read(),
headers={
'content-type': 'application/csv'
},
)
res = Resource(
title='Other remote file resource',
description='Other remote file resource',
link=httpserver.url,
format='csv',
openness_score=3,
views_count=10,
downloads_count=20,
dataset=buzzfeed_dataset,
created_by=buzzfeed_editor,
modified_by=buzzfeed_editor,
data_date=datetime.today()
)
res.save()
return res
def table_resource_with_invalid_schema(valid_dataset):
resource = Resource()
resource.url = "http://smth.smwhere.com"
resource.title = "File resource name"
resource.type = "file"
resource.format = 'XLSX'
resource.file = File(open(prepare_file('wrong_schema_table.xlsx'), 'rb'))
resource.file.open('rb')
resource.dataset = valid_dataset
resource.save()
return resource
def valid_resource_with_file(valid_dataset, file_csv):
resource = Resource()
resource.title = "File resource name"
resource.resource_type = "Table"
resource.type = "file"
resource.format = 'csv'
resource.file = File(file_csv)
resource.file.open('r')
resource.dataset = valid_dataset
resource.tracker.saved_data['link'] = resource.link
resource.save()
return resource
def valid_resource_with_description(valid_dataset):
resource = Resource()
resource.url = "http://smth.smwhere.com"
resource.title = "Resource name"
resource.description = "Test Resource Description"
resource.dataset = valid_dataset
resource.status = 'published'
resource.save()
return resource
def valid_resource(valid_dataset):
resource = Resource()
resource.link = "http://falconframework.org"
resource.title = "Resource name"
resource.resource_type = "Table"
resource.dataset = valid_dataset
resource.data_date = date.today()
resource.tracker.saved_data['link'] = resource.link
resource.save()
return resource
def valid_resource_with_description(valid_dataset):
resource = Resource()
resource.link = "http://falconframework.org"
resource.title = "Resource name"
resource.description = "Test Resource Description"
resource.dataset = valid_dataset
resource.status = 'published'
resource.tracker.saved_data['link'] = resource.link
resource.tracker.saved_data['file'] = resource.file
resource.save()
return resource
def resource_in_dataset_org2(dataset_org2):
resource = Resource()
resource.url = "http://tokill.shadowthieves.bg/cowled_wizards"
resource.title = "Zakapturzeni czarodzieje"
resource.resource_type = "Table"
resource.dataset = dataset_org2
resource.save()
return resource
def valid_resource2(valid_dataset2):
resource = Resource()
resource.url = "http://smth.smwhere.com"
resource.title = "Resource name2"
resource.resource_type = "Table"
resource.dataset = valid_dataset2
resource.save()
return resource
def removed_resource(valid_dataset2):
resource = Resource()
resource.link = "http://noth.nowhere.com"
resource.title = "removed resource"
resource.dataset = valid_dataset2
resource.is_removed = True
resource.save()
return resource
def draft_resource(valid_dataset2):
resource = Resource()
resource.link = "http://drafts.com"
resource.title = "draft resource"
resource.dataset = valid_dataset2
resource.status = "draft"
resource.save()
return resource
def resource_without_dataset(valid_dataset):
resource = Resource()
resource.url = "http://smth.smwhere.com"
resource.title = "Resource name"
resource.is_external = True
resource.resource_type = "Table"
# resource.dataset = valid_dataset
resource.save()
return resource
def valid_resource2(valid_dataset2):
resource = Resource()
resource.link = "http://falconframework.org"
resource.title = "Resource name2"
resource.resource_type = "Table"
resource.dataset = valid_dataset2
resource.tracker.saved_data['link'] = resource.link
resource.tracker.saved_data['file'] = resource.file
resource.save()
return resource
def resource_without_dataset(valid_dataset):
resource = Resource()
resource.link = "http://falconframework.org"
resource.title = "Resource name"
resource.is_external = True
resource.resource_type = "Table"
# resource.dataset = valid_dataset
resource.tracker.saved_data['link'] = resource.link
resource.tracker.saved_data['file'] = resource.file
resource.save()
return resource
def test_invalid_resources_comment(self, removed_resource, draft_resource,
client14):
invalid_resource = Resource()
invalid_resource.id = removed_resource.id + 100
for resource in [invalid_resource, removed_resource, draft_resource]:
resp = client14.simulate_post(
path=f'/resources/{resource.id}/comments',
json={
'data': {
'type': 'comment',
'attributes': {
'comment': "some valid\ncomment"
}
}
})
assert resp.status == HTTP_NOT_FOUND
assert resp.json['code'] == 'error'
assert resp.json['title'] == '404 Not Found'
def create_res(ds, editor, **kwargs):
from mcod.resources.models import Resource, ResourceFile
_fname = kwargs.pop('filename')
_kwargs = {
'title': 'Analysis of fake news sites and viral posts',
'description': 'Over the past four years, BuzzFeed News has maintained lists of sites that '
'publish completely fabricated stories. As we encounter new ones and debunk '
'their content, we add them to the list.',
'file': _fname,
'link': f'https://falconframework.org/media/resources/{_fname}',
'format': 'csv',
'openness_score': 3,
'views_count': 10,
'downloads_count': 20,
'dataset': ds,
'created_by': editor,
'modified_by': editor,
'data_date': datetime.today(),
}
if is_enabled('S41_resource_has_high_value_data'):
_kwargs['has_high_value_data'] = False
_kwargs.update(**kwargs)
if is_enabled('S40_new_file_model.be'):
with open(os.path.join(settings.RESOURCES_MEDIA_ROOT, _fname), 'rb') as f:
from mcod.resources.link_validation import session
adapter = requests_mock.Adapter()
adapter.register_uri('GET', _kwargs['link'], content=f.read(), headers={'Content-Type': 'application/csv'})
session.mount('https://falconframework.org', adapter)
_kwargs.pop('file')
res = Resource.objects.create(**_kwargs)
ResourceFile.objects.create(
is_main=True,
resource=res,
file=os.path.join(settings.RESOURCES_MEDIA_ROOT, _fname),
format='csv'
)
else:
res = Resource(**_kwargs)
res.tracker.saved_data['link'] = res.link
res.save()
res = Resource.objects.get(pk=res.pk)
return res
def no_data_resource(dataset):
resource = Resource()
resource.title = "No data resource"
resource.type = "file"
resource.format = 'JPG'
if not is_enabled('S40_new_file_model.be'):
resource.file = File(open(prepare_file('buzzfeed-logo.jpg'), 'rb'))
resource.file.open('rb')
resource.dataset = dataset
resource.save()
if is_enabled('S40_new_file_model.be'):
ResourceFile.objects.create(file=File(
open(prepare_file('buzzfeed-logo.jpg'), 'rb')),
is_main=True,
resource=resource,
format='JPG')
resource = Resource.objects.get(pk=resource.pk)
return resource
def remote_file_resource_of_api_type(buzzfeed_dataset, buzzfeed_editor, httpserver):
from mcod.resources.models import Resource
httpserver.serve_content(
content=get_json_file().read(),
headers={
'content-type': 'application/json'
},
)
res = Resource(
title='Remote file resource',
description='Remote file resource',
link=httpserver.url,
format='json',
openness_score=3,
views_count=10,
downloads_count=20,
dataset=buzzfeed_dataset,
created_by=buzzfeed_editor,
modified_by=buzzfeed_editor,
data_date=datetime.today(),
type='api',
)
res.save()
return res
def test_resource_create(self, valid_dataset):
r = Resource()
# r.name = "test"
r.title = "test"
r.description = "Opis zasobu"
r.format = "csv"
r.resource_type = "zestawienie"
r.old_resource_type = "zestawienie"
r.dataset = valid_dataset
r.link = "http://test.to.resource.pl/1.xls"
r.data_date = "2018-10-02"
assert r.full_clean() is None
assert r.id is None
r.save()
assert r.id is not None