def setUp(self): search.delete_all('test') source = "test" doc_id = 38 doc = { 'title': "TEST PROJECT", 'contributors': ['Me, Myself', 'And I'], 'properties': { 'description': 'science stuff', 'email': 'email stuff' }, 'meta': {}, 'id': doc_id, 'source': source, 'iso_timestamp': datetime.datetime.now().isoformat() } search.update(source, doc, 'article', doc_id)
def migrate(): try: search.delete_all('scrapi') except ElasticHttpNotFoundError: pass for dirname, dirnames, filenames in os.walk('archive/'): if os.path.isfile(dirname + '/normalized.json'): with open(dirname + '/normalized.json') as f: try: doc = json.load(f) except ValueError as e: logger.exception(e) continue try: search.update('scrapi', doc, dirname.split('/')[1], dirname.split('/')[2]) except ElasticHttpError as e: logger.exception(e) continue
def tearDown(self): search.delete_all('test')