def _rebuild(self, csv_file=None):
"""
Implement rebuild command
:param csv_file: path to .csv file for input
:type csv_file: str
:return: Nothing
:rtype: None
"""
self._clear_index()
conn = solr_connection(self.command_name)
lc = LocalCKAN()
if csv_file:
count = {}
for org_id, records in csv_data_batch(csv_file, self.command_name):
if org_id not in count:
count[org_id] = 0
org_detail = lc.action.organization_show(id=org_id)
_update_records(records, org_detail, conn, self.command_name)
count[org_id] += len(records)
for org_id in lc.action.organization_list():
print org_id, count.get(org_id, 0)
else:
for org in lc.action.organization_list():
count = 0
org_detail = lc.action.organization_show(id=org)
for records in data_batch(org_detail["id"], lc, self.command_name):
_update_records(records, org_detail, conn, self.command_name)
count += len(records)
print org, count
def _rebuild(self, csv_files=None):
"""
Implement rebuild command
:param csv_files: sequence of paths to .csv files for input
:type csv_files: sequence of str
:return: Nothing
:rtype: None
"""
self._clear_index()
conn = solr_connection("ati")
lc = LocalCKAN()
if csv_files:
for csv_file in csv_files:
print csv_file + ":"
for org_id, records in csv_data_batch(csv_file, TARGET_DATASET):
try:
org_detail = lc.action.organization_show(id=org_id)
except NotFound:
continue
print " {0:s} {1}".format(org_id, len(records))
_update_records(records, org_detail, conn)
else:
for org_id in lc.action.organization_list():
count = 0
org_detail = lc.action.organization_show(id=org_id)
for records in data_batch(org_detail["id"], lc, TARGET_DATASET):
_update_records(records, org_detail, conn)
count += len(records)
print org_id, count
def _rebuild(self, csv_file=None):
"""
Implement rebuild command
:param csv_file: path to .csv file for input
:type csv_file: str
:return: Nothing
:rtype: None
"""
conn = solr_connection('proactive_disclosure')
lc = LocalCKAN()
if csv_file:
count = {}
for org_recs in csv_data_batch(csv_file, TARGET_DATASET):
org_id = org_recs.keys()[0]
if org_id not in count:
count[org_id] = 0
org_detail = lc.action.organization_show(id=org_id)
records = org_recs[org_id]
_update_records(records, org_detail, conn)
count[org_id] += len(records)
for org_id in lc.action.organization_list():
print org_id, count.get(org_id, 0)
else:
for org in lc.action.organization_list():
count = 0
org_detail = lc.action.organization_show(id=org)
for records in data_batch(org_detail['id'], lc, TARGET_DATASET):
_update_records(records, org_detail, conn)
count += len(records)
print org, count
def _rebuild(self, csv_files=None):
"""
Implement rebuild command
:param csv_files: sequence of paths to .csv files for input
:type csv_files: sequence of str
:return: Nothing
:rtype: None
"""
self._clear_index()
conn = solr_connection('ati_summaries')
lc = LocalCKAN()
if csv_files:
count = {}
for csv_file in csv_files:
print csv_file + ':'
count[csv_file] = {}
for org_recs in csv_data_batch(csv_file, TARGET_DATASET):
org_id = org_recs.keys()[0]
if org_id not in count[csv_file]:
count[csv_file][org_id] = 0
try:
org_detail = lc.action.organization_show(id=org_id)
except NotFound:
continue
records = org_recs[org_id]
_update_records(records, org_detail, conn)
count[csv_file][org_id] += len(records)
for k, v in count[csv_file].iteritems():
print " {0:s} {1}".format(k, v)
for org_id in lc.action.organization_list():
print org_id, sum((count[f].get(org_id, 0) for f in count))
else:
for org_id in lc.action.organization_list():
count = 0
org_detail = lc.action.organization_show(id=org_id)
for records in data_batch(org_detail['id'], lc, TARGET_DATASET):
_update_records(records, org_detail, conn)
count += len(records)
print org_id, count
def rebuild(command_name, csv_files=None):
"""
Implement rebuild command
:param csv_file: path to .csv file for input
:type csv_file: str
:return: Nothing
:rtype: None
"""
clear_index(command_name)
conn = solr_connection(command_name)
lc = LocalCKAN()
if csv_files:
for csv_file in csv_files:
print csv_file + ':'
unmatched = None
for resource_name, org_id, records in csv_data_batch(csv_file, command_name):
try:
org_detail = lc.action.organization_show(id=org_id)
except NotFound:
continue
print " {0:s} {1}".format(org_id, len(records))
unmatched = _update_records(
records, org_detail, conn, resource_name, unmatched)
else:
for org in lc.action.organization_list():
count = 0
org_detail = lc.action.organization_show(id=org)
unmatched = None
for resource_name, records in data_batch(org_detail['id'], lc, command_name):
unmatched = _update_records(
records, org_detail, conn, resource_name, unmatched)
count += len(records)
print org, count