def delete(self, project_name):
if project_name not in data['projects']:
return rest.not_found()
try:
project_lock.acquire(project_name)
# - get corresponding domain
domain_name = data['projects'][project_name]['domain']
# - delete ingestion_id row from hbase updates table
# should we delete corresponding files on HDFS?
# delete hbase table sha1_infos?
ingestion_id = data['projects'][project_name]['ingestion_id']
from happybase.connection import Connection
conn = Connection(config['image']['hbase_host'])
table = conn.table(config['image']['hbase_table_updates'])
table.delete(ingestion_id, columns=['info:lopq_codes_path', 'info:lopq_model_pkl'])
# remove project:
# - from current data dict
del data['projects'][project_name]
# - files associated with project
shutil.rmtree(os.path.join(_get_project_dir_path(project_name)))
# - from mongodb
db_projects.delete_one({'project_name':project_name})
msg = 'project {} has been deleted'.format(project_name)
logger.info(msg)
# if it's the last project from a domain, shoud we remove the domain?
# for now assume one project per domain and delete too
# stop and remove docker container
docker_name = data['domains'][domain_name]['docker_name']
subproc = sub.Popen("sudo docker stop {}; sudo docker rm {}".format(docker_name, docker_name), shell=True)
# cleanup ports list
data['ports'].remove(data['domains'][domain_name]['port'])
# remove domain:
# - from current data dict
del data['domains'][domain_name]
# - files associated with project
shutil.rmtree(os.path.join(_get_domain_dir_path(domain_name)))
# - from mongodb
db_domains.delete_one({'domain_name':domain_name})
# should we also clean up things in HDFS?...
msg2 = 'domain {} has been deleted'.format(domain_name)
logger.info(msg2)
# regenerate apache conf from scratch for domains that are still active.
reset_apache_conf()
return rest.deleted(msg+' '+msg2)
except Exception as e:
logger.error('deleting project %s: %s' % (project_name, e.message))
return rest.internal_error('deleting project %s error, halted.' % project_name)
finally:
project_lock.remove(project_name)
def delete(self, project_name):
if project_name not in data['projects']:
return rest.not_found()
try:
project_lock.acquire(project_name)
# - get corresponding domain
domain_name = data['projects'][project_name]['domain']
# - delete ingestion_id row from hbase updates table
# should we delete corresponding files on HDFS?
# delete hbase table sha1_infos?
ingestion_id = data['projects'][project_name]['ingestion_id']
from happybase.connection import Connection
conn = Connection(config['image']['hbase_host'])
table = conn.table(config['image']['hbase_table_updates'])
table.delete(ingestion_id, columns=['info:lopq_codes_path', 'info:lopq_model_pkl'])
# remove project:
# - from current data dict
del data['projects'][project_name]
# - files associated with project
shutil.rmtree(os.path.join(_get_project_dir_path(project_name)))
# - from mongodb
db_projects.delete_one({'project_name':project_name})
msg = 'project {} has been deleted'.format(project_name)
logger.info(msg)
# if it's the last project from a domain, shoud we remove the domain?
# for now assume one project per domain and delete too
# stop and remove docker container
docker_name = data['domains'][domain_name]['docker_name']
subproc = sub.Popen("sudo docker stop {}; sudo docker rm {}".format(docker_name, docker_name), shell=True)
# cleanup ports list
data['ports'].remove(data['domains'][domain_name]['port'])
# remove domain:
# - from current data dict
del data['domains'][domain_name]
# - files associated with project
shutil.rmtree(os.path.join(_get_domain_dir_path(domain_name)))
# - from mongodb
db_domains.delete_one({'domain_name':domain_name})
# should we also clean up things in HDFS?...
msg2 = 'domain {} has been deleted'.format(domain_name)
logger.info(msg2)
# regenerate apache conf from scratch for domains that are still active.
reset_apache_conf()
return rest.deleted(msg+' '+msg2)
except Exception as e:
logger.error('deleting project %s: %s' % (project_name, e.message))
return rest.internal_error('deleting project %s error, halted.' % project_name)
finally:
project_lock.remove(project_name)
def process_event_query(self):
"""
This is the main function in this class. This calls several functions to validate input, set match clauses, set filter clauses
set sort clauses and finally resolve any nested documents if needed. Finally this function returns the data as a json or json_lines
joined by a '\n'
"""
valid_input = self.validate_input()
if not valid_input:
err_json = dict()
err_json['message'] = "Please enter valid query params. Fields must exist for the given project. " \
"If not sure, please access http://mydigurl/projects/<project_name>/fields " \
"API for reference"
return rest.bad_request(err_json)
try:
resp = self.cquery.process()
if resp[1] == 400:
logger.warning(
"Request generated 4xx response. Check request again")
return resp
else:
resp = resp[0]
if resp is not None and len(
resp['aggregations'][self.field]['buckets']) > 0:
if "." not in self.field and self.config[
self.field]['type'] == "date":
ts, dims = DigOutputProcessor(
resp['aggregations'][self.field], self.agg_field,
True).process()
ts_obj = TimeSeries(
ts, {},
dims,
percent_change=self.percent_change,
impute_method=self.impute_method).to_dict()
else:
ts, dims = DigOutputProcessor(
resp['aggregations'][self.field], self.agg_field,
False).process()
ts_obj = TimeSeries(
ts, {},
dims,
percent_change=self.percent_change,
impute_method=self.impute_method).to_dict()
return rest.ok(ts_obj)
else:
return rest.not_found("No Time series found for query")
except Exception as e:
logger.exception(
"Exception encountered while performing Event query")
return rest.internal_error("Internal Error occured")
def post(self):
input = request.get_json(force=True)
logger.info('/projects received: %s' % (input))
project_name = input.get('project_name', '')
if len(project_name) == 0 or len(project_name) >= 256:
return rest.bad_request('Invalid project name.')
if project_name in data['projects']:
#msg = 'You cannot post an existing project to the /projects endpoint. For updates, post to projects/{your_project_name}'
msg = 'You cannot post an existing project to the /projects endpoint.'
return rest.bad_request(msg)
project_sources = input.get('sources', [])
if len(project_sources) == 0:
return rest.bad_request('Invalid sources.')
logger.info('/projects project_name: %s' % (project_name))
logger.info('/projects project_sources: %s' % (project_sources))
try:
# create project data structure, folders & files
project_dir_path = _get_project_dir_path(project_name)
project_lock.acquire(project_name)
logger.info('/projects creating directory: %s' % (project_dir_path))
os.makedirs(project_dir_path)
data['projects'][project_name] = {'sources': {}}
data['projects'][project_name]['project_name'] = project_name
data['projects'][project_name]['sources'] = project_sources
with open(os.path.join(project_dir_path, 'project_config.json'), 'w') as f:
f.write(json.dumps(data['projects'][project_name], indent=4, default=json_encode))
# we should try to create a service for domain "sources:type"
# (or update it if timerange defined by "sources:start_date" and "sources:end_date" is bigger than existing)
ret, domain_name, ingestion_id, job_id, err = check_domain_service(project_sources, project_name)
data['projects'][project_name]['domain'] = domain_name
if ret==0:
msg = 'project %s created.' % project_name
logger.info(msg)
# store job infos
data['projects'][project_name]['ingestion_id'] = ingestion_id
data['projects'][project_name]['job_id'] = job_id
data['projects'][project_name]['status'] = 'indexing'
# insert into mongoDB
logger.info('Project %s (before mongodb insertion) dict keys are %s' % (project_name, data['projects'][project_name].keys()))
db_projects.insert_one(data['projects'][project_name])
logger.info('Project %s (after mongodb insertion) dict keys are %s' % (project_name, data['projects'][project_name].keys()))
# How come data['projects'][project_name] has an '_id' field now???
if '_id' in data['projects'][project_name]:
del data['projects'][project_name]['_id']
logger.info('Project %s (after mongodb insertion and cleaning) dict keys are %s' % (project_name, data['projects'][project_name].keys()))
try:
return rest.created(msg)
finally:
restart_apache()
elif ret==1:
msg = 'domain for project %s was already previously created. %s' % (project_name, err)
logger.info(msg)
# what should we return in this case
return rest.ok(msg)
else:
# we should remove project_name
del data['projects'][project_name]
msg = 'project %s creation failed while creating search service: %s' % (project_name, err)
logger.info(msg)
return rest.internal_error(msg)
except Exception as e:
# try to remove project_name
try:
del data['projects'][project_name]
except:
pass
# try to remove data files too
try:
shutil.rmtree(os.path.join(_get_project_dir_path(project_name)))
except:
pass
msg = 'project {} creation failed: {} {}'.format(project_name, e, sys.exc_info()[0])
logger.error(msg)
return rest.internal_error(msg)
finally:
project_lock.release(project_name)
def post(self):
input = request.get_json(force=True)
logger.info('/projects received: %s' % (input))
project_name = input.get('project_name', '')
if len(project_name) == 0 or len(project_name) >= 256:
return rest.bad_request('Invalid project name.')
if project_name in data['projects']:
#msg = 'You cannot post an existing project to the /projects endpoint. For updates, post to projects/{your_project_name}'
msg = 'You cannot post an existing project to the /projects endpoint.'
return rest.bad_request(msg)
project_sources = input.get('sources', [])
if len(project_sources) == 0:
return rest.bad_request('Invalid sources.')
logger.info('/projects project_name: %s' % (project_name))
logger.info('/projects project_sources: %s' % (project_sources))
try:
# create project data structure, folders & files
project_dir_path = _get_project_dir_path(project_name)
project_lock.acquire(project_name)
logger.info('/projects creating directory: %s' % (project_dir_path))
os.makedirs(project_dir_path)
data['projects'][project_name] = {'sources': {}}
data['projects'][project_name]['project_name'] = project_name
data['projects'][project_name]['sources'] = project_sources
with open(os.path.join(project_dir_path, 'project_config.json'), 'w') as f:
f.write(json.dumps(data['projects'][project_name], indent=4, default=json_encode))
# we should try to create a service for domain "sources:type"
# (or update it if timerange defined by "sources:start_date" and "sources:end_date" is bigger than existing)
ret, domain_name, ingestion_id, job_id, err = check_domain_service(project_sources, project_name)
data['projects'][project_name]['domain'] = domain_name
if ret==0:
msg = 'project %s created.' % project_name
logger.info(msg)
# store job infos
data['projects'][project_name]['ingestion_id'] = ingestion_id
data['projects'][project_name]['job_id'] = job_id
data['projects'][project_name]['status'] = 'indexing'
# insert into mongoDB
logger.info('Project %s (before mongodb insertion) dict keys are %s' % (project_name, data['projects'][project_name].keys()))
db_projects.insert_one(data['projects'][project_name])
logger.info('Project %s (after mongodb insertion) dict keys are %s' % (project_name, data['projects'][project_name].keys()))
# How come data['projects'][project_name] has an '_id' field now???
if '_id' in data['projects'][project_name]:
del data['projects'][project_name]['_id']
logger.info('Project %s (after mongodb insertion and cleaning) dict keys are %s' % (project_name, data['projects'][project_name].keys()))
try:
return rest.created(msg)
finally:
restart_apache()
elif ret==1:
msg = 'domain for project %s was already previously created. %s' % (project_name, err)
logger.info(msg)
# what should we return in this case
return rest.ok(msg)
else:
# we should remove project_name
del data['projects'][project_name]
msg = 'project %s creation failed while creating search service: %s' % (project_name, err)
logger.info(msg)
return rest.internal_error(msg)
except Exception as e:
# try to remove project_name
try:
del data['projects'][project_name]
except:
pass
# try to remove data files too
try:
shutil.rmtree(os.path.join(_get_project_dir_path(project_name)))
except:
pass
msg = 'project {} creation failed: {} {}'.format(project_name, e, sys.exc_info()[0])
logger.error(msg)
return rest.internal_error(msg)
finally:
project_lock.release(project_name)