def buzzfeed_fakenews_resource(buzzfeed_dataset, buzzfeed_editor, mocker): from mcod.resources.models import Resource _name = 'buzzfeed-2018-fake-news-1000-lines.csv' copyfile(os.path.join(settings.TEST_DATA_PATH, _name), os.path.join(settings.RESOURCES_MEDIA_ROOT, _name)) res = Resource( title='Analysis of fake news sites and viral posts', description= 'Over the past four years, BuzzFeed News has maintained lists of sites that ' 'publish completely fabricated stories. As we encounter new ones and debunk ' 'their content, we add them to the list.', file=_name, link=f'http://falconframework.org/media/resources/{_name}', format='csv', openness_score=3, views_count=10, downloads_count=20, dataset=buzzfeed_dataset, created_by=buzzfeed_editor, modified_by=buzzfeed_editor, data_date=datetime.today()) res.tracker.saved_data['link'] = res.link res.save() return res
def other_remote_file_resource(buzzfeed_dataset, buzzfeed_editor, mocker, httpserver): from mcod.resources.models import Resource simple_csv_path = os.path.join(settings.TEST_SAMPLES_PATH, 'simple.csv') httpserver.serve_content( content=open(simple_csv_path).read(), headers={ 'content-type': 'application/csv' }, ) res = Resource( title='Other remote file resource', description='Other remote file resource', link=httpserver.url, format='csv', openness_score=3, views_count=10, downloads_count=20, dataset=buzzfeed_dataset, created_by=buzzfeed_editor, modified_by=buzzfeed_editor, data_date=datetime.today() ) res.save() return res
def table_resource_with_invalid_schema(valid_dataset): resource = Resource() resource.url = "http://smth.smwhere.com" resource.title = "File resource name" resource.type = "file" resource.format = 'XLSX' resource.file = File(open(prepare_file('wrong_schema_table.xlsx'), 'rb')) resource.file.open('rb') resource.dataset = valid_dataset resource.save() return resource
def valid_resource_with_file(valid_dataset, file_csv): resource = Resource() resource.title = "File resource name" resource.resource_type = "Table" resource.type = "file" resource.format = 'csv' resource.file = File(file_csv) resource.file.open('r') resource.dataset = valid_dataset resource.tracker.saved_data['link'] = resource.link resource.save() return resource
def valid_resource_with_description(valid_dataset): resource = Resource() resource.url = "http://smth.smwhere.com" resource.title = "Resource name" resource.description = "Test Resource Description" resource.dataset = valid_dataset resource.status = 'published' resource.save() return resource
def valid_resource(valid_dataset): resource = Resource() resource.link = "http://falconframework.org" resource.title = "Resource name" resource.resource_type = "Table" resource.dataset = valid_dataset resource.data_date = date.today() resource.tracker.saved_data['link'] = resource.link resource.save() return resource
def valid_resource_with_description(valid_dataset): resource = Resource() resource.link = "http://falconframework.org" resource.title = "Resource name" resource.description = "Test Resource Description" resource.dataset = valid_dataset resource.status = 'published' resource.tracker.saved_data['link'] = resource.link resource.tracker.saved_data['file'] = resource.file resource.save() return resource
def resource_in_dataset_org2(dataset_org2): resource = Resource() resource.url = "http://tokill.shadowthieves.bg/cowled_wizards" resource.title = "Zakapturzeni czarodzieje" resource.resource_type = "Table" resource.dataset = dataset_org2 resource.save() return resource
def valid_resource2(valid_dataset2): resource = Resource() resource.url = "http://smth.smwhere.com" resource.title = "Resource name2" resource.resource_type = "Table" resource.dataset = valid_dataset2 resource.save() return resource
def removed_resource(valid_dataset2): resource = Resource() resource.link = "http://noth.nowhere.com" resource.title = "removed resource" resource.dataset = valid_dataset2 resource.is_removed = True resource.save() return resource
def draft_resource(valid_dataset2): resource = Resource() resource.link = "http://drafts.com" resource.title = "draft resource" resource.dataset = valid_dataset2 resource.status = "draft" resource.save() return resource
def resource_without_dataset(valid_dataset): resource = Resource() resource.url = "http://smth.smwhere.com" resource.title = "Resource name" resource.is_external = True resource.resource_type = "Table" # resource.dataset = valid_dataset resource.save() return resource
def valid_resource2(valid_dataset2): resource = Resource() resource.link = "http://falconframework.org" resource.title = "Resource name2" resource.resource_type = "Table" resource.dataset = valid_dataset2 resource.tracker.saved_data['link'] = resource.link resource.tracker.saved_data['file'] = resource.file resource.save() return resource
def resource_without_dataset(valid_dataset): resource = Resource() resource.link = "http://falconframework.org" resource.title = "Resource name" resource.is_external = True resource.resource_type = "Table" # resource.dataset = valid_dataset resource.tracker.saved_data['link'] = resource.link resource.tracker.saved_data['file'] = resource.file resource.save() return resource
def test_invalid_resources_comment(self, removed_resource, draft_resource, client14): invalid_resource = Resource() invalid_resource.id = removed_resource.id + 100 for resource in [invalid_resource, removed_resource, draft_resource]: resp = client14.simulate_post( path=f'/resources/{resource.id}/comments', json={ 'data': { 'type': 'comment', 'attributes': { 'comment': "some valid\ncomment" } } }) assert resp.status == HTTP_NOT_FOUND assert resp.json['code'] == 'error' assert resp.json['title'] == '404 Not Found'
def create_res(ds, editor, **kwargs): from mcod.resources.models import Resource, ResourceFile _fname = kwargs.pop('filename') _kwargs = { 'title': 'Analysis of fake news sites and viral posts', 'description': 'Over the past four years, BuzzFeed News has maintained lists of sites that ' 'publish completely fabricated stories. As we encounter new ones and debunk ' 'their content, we add them to the list.', 'file': _fname, 'link': f'https://falconframework.org/media/resources/{_fname}', 'format': 'csv', 'openness_score': 3, 'views_count': 10, 'downloads_count': 20, 'dataset': ds, 'created_by': editor, 'modified_by': editor, 'data_date': datetime.today(), } if is_enabled('S41_resource_has_high_value_data'): _kwargs['has_high_value_data'] = False _kwargs.update(**kwargs) if is_enabled('S40_new_file_model.be'): with open(os.path.join(settings.RESOURCES_MEDIA_ROOT, _fname), 'rb') as f: from mcod.resources.link_validation import session adapter = requests_mock.Adapter() adapter.register_uri('GET', _kwargs['link'], content=f.read(), headers={'Content-Type': 'application/csv'}) session.mount('https://falconframework.org', adapter) _kwargs.pop('file') res = Resource.objects.create(**_kwargs) ResourceFile.objects.create( is_main=True, resource=res, file=os.path.join(settings.RESOURCES_MEDIA_ROOT, _fname), format='csv' ) else: res = Resource(**_kwargs) res.tracker.saved_data['link'] = res.link res.save() res = Resource.objects.get(pk=res.pk) return res
def no_data_resource(dataset): resource = Resource() resource.title = "No data resource" resource.type = "file" resource.format = 'JPG' if not is_enabled('S40_new_file_model.be'): resource.file = File(open(prepare_file('buzzfeed-logo.jpg'), 'rb')) resource.file.open('rb') resource.dataset = dataset resource.save() if is_enabled('S40_new_file_model.be'): ResourceFile.objects.create(file=File( open(prepare_file('buzzfeed-logo.jpg'), 'rb')), is_main=True, resource=resource, format='JPG') resource = Resource.objects.get(pk=resource.pk) return resource
def remote_file_resource_of_api_type(buzzfeed_dataset, buzzfeed_editor, httpserver): from mcod.resources.models import Resource httpserver.serve_content( content=get_json_file().read(), headers={ 'content-type': 'application/json' }, ) res = Resource( title='Remote file resource', description='Remote file resource', link=httpserver.url, format='json', openness_score=3, views_count=10, downloads_count=20, dataset=buzzfeed_dataset, created_by=buzzfeed_editor, modified_by=buzzfeed_editor, data_date=datetime.today(), type='api', ) res.save() return res
def test_resource_create(self, valid_dataset): r = Resource() # r.name = "test" r.title = "test" r.description = "Opis zasobu" r.format = "csv" r.resource_type = "zestawienie" r.old_resource_type = "zestawienie" r.dataset = valid_dataset r.link = "http://test.to.resource.pl/1.xls" r.data_date = "2018-10-02" assert r.full_clean() is None assert r.id is None r.save() assert r.id is not None