def set_discrete_gpu_state(self, state: bool) -> None:
super().set_discrete_gpu_state(state)
if state:
self.patch_file_with_pci_id(
self.get_config_file('nouveau-xorg.conf'), xorg_file)
else:
utils.remove(xorg_file)
def reset(self):
self.content = None
self.code = 1
if self.inputPath is not None:
remove(self.inputPath)
def retrieve(self):
"""
retrieve(self)
Compares files uploaded(json files in git) to objects already on disk
to determine which files to download. File on disk that are not in git are
removed.
"""
self.logger.info('Beginning to retrieve files.')
objects_to_delete = []
if not 'pitem' in self.options:
# Sync non-binary files in shelves to dest_sync
self.sync_local_files()
# Delete objects in dest_sync on disk
objects_to_delete = self.get_objects_to_delete()
if objects_to_delete:
utils.remove(objects_to_delete)
# Download objects
objects_to_retrieve = self.get_objects_to_retrieve()
if objects_to_retrieve:
with FB_ObjectStore(self.options) as objectstore:
objectstore.download(objects_to_retrieve)
else:
self.logger.info('Repo up-to-date. No files to retrieve.')
self.write_updated_objects_to_file(objects_to_retrieve,
objects_to_delete)
self.logger.info('Finished')
def _install_influxdb():
influxdb_source_url = ctx_properties['influxdb_rpm_source_url']
influxdb_user = '******'
influxdb_group = 'influxdb'
influxdb_home = '/opt/influxdb'
influxdb_log_path = '/var/log/cloudify/influxdb'
ctx.logger.info('Installing InfluxDB...')
utils.set_selinux_permissive()
utils.copy_notice(INFLUX_SERVICE_NAME)
utils.mkdir(influxdb_home)
utils.mkdir(influxdb_log_path)
utils.yum_install(influxdb_source_url, service_name=INFLUX_SERVICE_NAME)
utils.sudo(['rm', '-rf', '/etc/init.d/influxdb'])
ctx.logger.info('Deploying InfluxDB config.toml...')
utils.deploy_blueprint_resource(
'{0}/config.toml'.format(CONFIG_PATH),
'{0}/shared/config.toml'.format(influxdb_home), INFLUX_SERVICE_NAME)
ctx.logger.info('Fixing user permissions...')
utils.chown(influxdb_user, influxdb_group, influxdb_home)
utils.chown(influxdb_user, influxdb_group, influxdb_log_path)
utils.systemd.configure(INFLUX_SERVICE_NAME)
# Provided with InfluxDB's package. Will be removed if it exists.
utils.remove('/etc/init.d/influxdb')
utils.logrotate(INFLUX_SERVICE_NAME)
def set_discrete_gpu_state(self, state: bool) -> None:
"""
Patch the system to use discrete GPU as primary GPU or not
:param state: true for performance mode (Discrete GPU as primary), false for power saving mode
"""
# Manjaro specific fix
utils.remove("/etc/X11/xorg.conf.d/90-mhwd.conf")
def _configure_influxdb(host, port):
db_user = "******"
db_pass = "******"
db_name = "cloudify"
ctx.logger.info('Creating InfluxDB Database...')
# the below request is equivalent to running:
# curl -S -s "http://localhost:8086/db?u=root&p=root" '-d "{\"name\": \"cloudify\"}" # NOQA
import urllib
import urllib2
import ast
endpoint_for_list = 'http://{0}:{1}/db'.format(host, port)
endpoint_for_creation = ('http://{0}:{1}/cluster/database_configs/'
'{2}'.format(host, port, db_name))
params = urllib.urlencode(dict(u=db_user, p=db_pass))
url_for_list = endpoint_for_list + '?' + params
url_for_creation = endpoint_for_creation + '?' + params
# check if db already exists
db_list = eval(urllib2.urlopen(urllib2.Request(url_for_list)).read())
try:
assert not any(d.get('name') == db_name for d in db_list)
except AssertionError:
ctx.logger.info('Database {0} already exists!'.format(db_name))
return
try:
utils.deploy_blueprint_resource(
'{0}/retention.json'.format(CONFIG_PATH),
'/tmp/retention.json', INFLUX_SERVICE_NAME)
with open('/tmp/retention.json') as policy_file:
retention_policy = policy_file.read()
ctx.logger.debug(
'Using retention policy: \n{0}'.format(retention_policy))
data = json.dumps(ast.literal_eval(retention_policy))
ctx.logger.debug('Using retention policy: \n{0}'.format(data))
content_length = len(data)
request = urllib2.Request(url_for_creation, data, {
'Content-Type': 'application/json',
'Content-Length': content_length})
ctx.logger.debug('Request is: {0}'.format(request))
request_reader = urllib2.urlopen(request)
response = request_reader.read()
ctx.logger.debug('Response: {0}'.format(response))
request_reader.close()
utils.remove('/tmp/retention.json')
except Exception as ex:
ctx.abort_operation('Failed to create: {0} ({1}).'.format(db_name, ex))
# verify db created
ctx.logger.info('Verifying database create successfully...')
db_list = eval(urllib2.urlopen(urllib2.Request(url_for_list)).read())
try:
assert any(d.get('name') == db_name for d in db_list)
except AssertionError:
ctx.abort_operation('Verification failed!')
ctx.logger.info('Databased {0} created successfully.'.format(db_name))
def _install_influxdb():
influxdb_source_url = ctx_properties['influxdb_rpm_source_url']
influxdb_user = '******'
influxdb_group = 'influxdb'
ctx.logger.info('Installing InfluxDB...')
utils.set_selinux_permissive()
utils.copy_notice(SERVICE_NAME)
utils.mkdir(HOME_DIR)
utils.mkdir(LOG_DIR)
utils.yum_install(influxdb_source_url, service_name=SERVICE_NAME)
ctx.logger.info('Deploying InfluxDB configuration...')
utils.deploy_blueprint_resource('{0}/config.toml'.format(CONFIG_PATH),
'{0}/shared/config.toml'.format(HOME_DIR),
SERVICE_NAME)
utils.chown(influxdb_user, influxdb_group, HOME_DIR)
utils.chown(influxdb_user, influxdb_group, LOG_DIR)
utils.systemd.configure(SERVICE_NAME)
# Provided with InfluxDB's package. Will be removed if it exists.
utils.remove(INIT_D_PATH)
utils.logrotate(SERVICE_NAME)
def link(self, src, dst):
'''Sets src as the link target of dst'''
src = os.path.join(self.base, os.path.expanduser(src))
dst = os.path.join(self.base, os.path.expanduser(dst))
if os.path.exists(dst):
if os.path.realpath(dst) == os.path.realpath(src):
self.log.sub('ignoring "%s", link exists' % dst)
return []
if self.dry:
self.log.dry('would remove %s and link it to %s'
% (dst, src))
return []
if self.safe and \
not self.log.ask('Remove "%s" for link creation?' % dst):
self.log.warn('ignoring "%s", link was not created' % dst)
return []
try:
utils.remove(dst)
except OSError:
self.log.err('something went wrong with %s' % src)
return []
if self.dry:
self.log.dry('would link %s to %s' % (dst, src))
return []
os.symlink(src, dst)
self.log.sub('linked %s to %s' % (dst, src))
# Follows original developer's behavior
return [(src, dst)]
def _create_db_tables_and_add_defaults():
ctx.logger.info('Creating SQL tables and adding default values...')
script_name = 'create_tables_and_add_defaults.py'
source_script_path = join('components/restservice/config', script_name)
destination_script_path = join(tempfile.gettempdir(), script_name)
ctx.download_resource(source_script_path, destination_script_path)
args_dict = runtime_props['security_configuration']
args_dict['amqp_host'] = runtime_props['rabbitmq_endpoint_ip']
args_dict['amqp_username'] = runtime_props['rabbitmq_username']
args_dict['amqp_password'] = runtime_props['rabbitmq_password']
args_dict['postgresql_host'] = runtime_props['postgresql_host']
args_dict['db_migrate_dir'] = join(
utils.MANAGER_RESOURCES_HOME,
'cloudify',
'migrations'
)
# The script won't have access to the ctx, so we dump the relevant args
# to a JSON file, and pass its path to the script
args_file_location = join(tempfile.gettempdir(), 'security_config.json')
with open(args_file_location, 'w') as f:
json.dump(args_dict, f)
# Directly calling with this python bin, in order to make sure it's run
# in the correct venv
python_path = join(runtime_props['home_dir'], 'env', 'bin', 'python')
result = utils.sudo(
[python_path, destination_script_path, args_file_location]
)
_log_results(result)
utils.remove(args_file_location)
utils.remove(destination_script_path)
def _update_pairwise_similarity(self, topic_id, content, date):
"""
updates similarity data within the corpus
"""
bow = self.dictionary.doc2bow(content)
for tid, data in self.data.items():
day_delta = (int(date) - int(data['date'])) / NUM_SECONDS_PER_DAY
time_factor = math.pow(self.time_decay, day_delta)
if tid == topic_id:
continue
bow1 = self.dictionary.doc2bow(data['body'])
sim = matutils.cossim(bow, bow1)
sim_1 = sim * min(1.0, 1 / time_factor)
sim_2 = sim * min(1.0, time_factor)
if self.irrelevant_thresh <= sim_1 <= self.duplicate_thresh:
del_id = insert(data['sim_list'], topic_id, sim_1,
self.max_recoms)
if del_id is not None:
self.data[topic_id]['appears_in'].append(tid)
self.data[tid]['updated'] = True
if del_id != '':
remove(self.data[del_id]['appears_in'], tid)
if self.irrelevant_thresh <= sim_2 <= self.duplicate_thresh:
del_id = insert(self.data[topic_id]['sim_list'], tid, sim_2,
self.max_recoms)
if del_id is not None:
self.data[tid]['appears_in'].append(topic_id)
if del_id != '':
remove(self.data[del_id]['appears_in'], topic_id)
def _install_influxdb():
influxdb_source_url = ctx_properties['influxdb_rpm_source_url']
influxdb_user = '******'
influxdb_group = 'influxdb'
influxdb_home = '/opt/influxdb'
influxdb_log_path = '/var/log/cloudify/influxdb'
ctx.logger.info('Installing InfluxDB...')
utils.set_selinux_permissive()
utils.copy_notice(INFLUX_SERVICE_NAME)
utils.mkdir(influxdb_home)
utils.mkdir(influxdb_log_path)
utils.yum_install(influxdb_source_url, service_name=INFLUX_SERVICE_NAME)
utils.sudo(['rm', '-rf', '/etc/init.d/influxdb'])
ctx.logger.info('Deploying InfluxDB config.toml...')
utils.deploy_blueprint_resource(
'{0}/config.toml'.format(CONFIG_PATH),
'{0}/shared/config.toml'.format(influxdb_home),
INFLUX_SERVICE_NAME)
ctx.logger.info('Fixing user permissions...')
utils.chown(influxdb_user, influxdb_group, influxdb_home)
utils.chown(influxdb_user, influxdb_group, influxdb_log_path)
utils.systemd.configure(INFLUX_SERVICE_NAME)
# Provided with InfluxDB's package. Will be removed if it exists.
utils.remove('/etc/init.d/influxdb')
utils.logrotate(INFLUX_SERVICE_NAME)
def _create_db_tables_and_add_users():
ctx.logger.info('Creating SQL tables and adding admin users...')
create_script_path = 'components/restservice/config' \
'/create_tables_and_add_users.py'
create_script_destination = join(tempfile.gettempdir(),
'create_tables_and_add_users.py')
ctx.download_resource(source=create_script_path,
destination=create_script_destination)
# Directly calling with this python bin, in order to make sure it's run
# in the correct venv
python_path = '{0}/env/bin/python'.format(REST_SERVICE_HOME)
runtime_props = ctx.instance.runtime_properties
args_dict = json.loads(runtime_props['security_configuration'])
args_dict['postgresql_host'] = runtime_props['postgresql_host']
# The script won't have access to the ctx, so we dump the relevant args
# to a JSON file, and pass its path to the script
args_file_location = join(tempfile.gettempdir(), 'security_config.json')
with open(args_file_location, 'w') as f:
json.dump(args_dict, f)
result = utils.sudo(
[python_path, create_script_destination, args_file_location]
)
_log_results(result)
utils.remove(args_file_location)
def _create_db_tables_and_add_defaults():
ctx.logger.info('Creating SQL tables and adding default values...')
script_name = 'create_tables_and_add_defaults.py'
source_script_path = join('components/restservice/config', script_name)
destination_script_path = join(tempfile.gettempdir(), script_name)
ctx.download_resource(source_script_path, destination_script_path)
args_dict = runtime_props['security_configuration']
args_dict['amqp_host'] = runtime_props['rabbitmq_endpoint_ip']
args_dict['amqp_username'] = ctx_properties['rabbitmq_username']
args_dict['amqp_password'] = ctx_properties['rabbitmq_password']
args_dict['postgresql_host'] = ctx_properties['postgresql_host']
args_dict['authorization_file_path'] = \
runtime_props['authorization_file_path']
args_dict['db_migrate_dir'] = join(utils.MANAGER_RESOURCES_HOME,
'cloudify', 'migrations')
# The script won't have access to the ctx, so we dump the relevant args
# to a JSON file, and pass its path to the script
args_file_location = join(tempfile.gettempdir(), 'security_config.json')
with open(args_file_location, 'w') as f:
json.dump(args_dict, f)
# Directly calling with this python bin, in order to make sure it's run
# in the correct venv
python_path = join(runtime_props['home_dir'], 'env', 'bin', 'python')
result = utils.sudo(
[python_path, destination_script_path, args_file_location])
_log_results(result)
utils.remove(args_file_location)
utils.remove(destination_script_path)
def retrieve(self):
"""
retrieve(self)
Compares files uploaded(json files in git) to objects already on disk
to determine which files to download. File on disk that are not in git are
removed.
"""
self.logger.info('Beginning to retrieve files.')
objects_to_delete = []
if not 'pitem' in self.options:
# Sync non-binary files in shelves to dest_sync
self.sync_local_files()
# Delete objects in dest_sync on disk
objects_to_delete = self.get_objects_to_delete()
if objects_to_delete:
utils.remove(objects_to_delete)
# Download objects
objects_to_retrieve = self.get_objects_to_retrieve()
if objects_to_retrieve:
with FB_ObjectStore(self.options) as objectstore:
objectstore.download(objects_to_retrieve)
else:
self.logger.info('Repo up-to-date. No files to retrieve.')
self.write_updated_objects_to_file(
objects_to_retrieve,
objects_to_delete
)
self.logger.info('Finished')
def _install_stage():
nodejs_source_url = ctx_properties['nodejs_tar_source_url']
stage_source_url = ctx_properties['stage_tar_source_url']
if not utils.resource_factory.local_resource_exists(stage_source_url):
ctx.logger.info('Stage package not found in manager resources '
'package. Stage will not be installed.')
ctx.instance.runtime_properties['skip_installation'] = 'true'
return
# injected as an input to the script
ctx.instance.runtime_properties['influxdb_endpoint_ip'] = \
os.environ.get('INFLUXDB_ENDPOINT_IP')
utils.set_selinux_permissive()
utils.copy_notice(SERVICE_NAME)
utils.mkdir(NODEJS_DIR)
utils.mkdir(HOME_DIR)
utils.mkdir(LOG_DIR)
utils.create_service_user(STAGE_USER, STAGE_GROUP, HOME_DIR)
ctx.logger.info('Installing NodeJS...')
nodejs = utils.download_cloudify_resource(nodejs_source_url, SERVICE_NAME)
utils.untar(nodejs, NODEJS_DIR)
utils.remove(nodejs)
ctx.logger.info('Installing Cloudify Stage (UI)...')
stage_tar = utils.download_cloudify_resource(stage_source_url,
SERVICE_NAME)
if 'community' in stage_tar:
ctx.logger.info('Community edition')
ctx.instance.runtime_properties['community_mode'] = '-mode community'
else:
ctx.instance.runtime_properties['community_mode'] = ''
utils.untar(stage_tar, HOME_DIR)
utils.remove(stage_tar)
ctx.logger.info('Fixing permissions...')
utils.chown(STAGE_USER, STAGE_GROUP, HOME_DIR)
utils.chown(STAGE_USER, STAGE_GROUP, NODEJS_DIR)
utils.chown(STAGE_USER, STAGE_GROUP, LOG_DIR)
utils.deploy_sudo_command_script(
'restore-snapshot.py',
'Restore stage directories from a snapshot path',
component=SERVICE_NAME,
allow_as=STAGE_USER)
utils.chmod('a+rx', '/opt/cloudify/stage/restore-snapshot.py')
utils.sudo(['usermod', '-aG', utils.CLOUDIFY_GROUP, STAGE_USER])
utils.logrotate(SERVICE_NAME)
utils.systemd.configure(SERVICE_NAME)
backend_dir = join(HOME_DIR, 'backend')
npm_path = join(NODEJS_DIR, 'bin', 'npm')
subprocess.check_call('cd {0}; {1} run db-migrate'.format(
backend_dir, npm_path),
shell=True)
def callback(self, event, data, record, skip):
if self.stop:
return False
if event == CONNECT:
return True
if event == DATA:
if skip[0]:
skipped = min(len(data), skip[0])
data = data[skipped:]
skip[0] -= skipped
if data:
self.wfile.write(data)
return True
if event == CLOSED:
record.numFailures = 0
record.sent = time.time()
if os.path.exists(record.partialPath):
utils.remove(record.partialPath)
self.wfile.close()
return False
if event == FAILURE:
record.numFailures += 1
message = 'Download of enclosure file %s failed: %s' % (record.url, data)
self.server.display.error(message)
# Write bogus text to wfile. Hopefully, it will cause iTunes to abort.
self.wfile.write('error: ' + message)
self.wfile.close()
return False
def update_on_delete_topic(self, topic_id):
if topic_id not in self.target_corpus.data:
return
for tid in self.target_corpus.data[topic_id]['appears_in_special']:
if topic_id in self.data[tid]['recommendations']:
remove(self.data[tid]['recommendations'], topic_id)
def generate(self,
proc_name,
proc_id,
proc_dict,
integration_grids,
analysis=''):
purpose = proc_dict['purpose']
options = proc_dict.get('whizard_options', '--no-banner')
sindarin = proc_name + '.sin'
runfolder = proc_name + '-' + str(proc_id)
fifo = proc_name + '-' + str(proc_id) + '.hepmc'
event_generation = purpose == 'events' or purpose == 'histograms'
only_sindarins = proc_dict.get('only_sindarins', False)
ut.mkdirs(runfolder)
if not only_sindarins:
_exe = lambda p: self.execute(p,
sindarin,
fifo=fifo,
proc_id=proc_id,
options=options,
analysis=analysis)
else:
_exe = lambda p: SUCCESS
if event_generation:
shutil.copyfile(sindarin, os.path.join(runfolder, sindarin))
if not ut.grep('grid_path', sindarin):
shutil.copyfile(integration_grids,
os.path.join(runfolder, integration_grids))
with ut.cd(runfolder):
if (purpose == 'histograms'):
ut.remove(
fifo
) # Suppress annoying display output if Fifo is already present
if not os.path.isfile(fifo):
subprocess.call("mkfifo " + fifo, shell=True)
change_sindarin_for_event_gen(sindarin, runfolder, proc_id,
proc_dict)
return _exe(purpose)
elif purpose == 'scan':
prepare_scan_sindarin(proc_name, proc_id, proc_dict, runfolder,
sindarin)
with ut.cd(runfolder):
set_seed(sindarin, runfolder)
return _exe(purpose)
elif purpose == 'test_soft':
options = options + ' --debug subtraction '
replace_line = lambda line: line.replace('include("',
'include("../')
integration_sindarin = proc_name + '-integrate.sin'
target_sindarin = os.path.join(runfolder, sindarin)
ut.sed(integration_sindarin,
replace_line=replace_line,
write_to_top='?test_soft_limit = true\n',
new_file=target_sindarin)
replace_nlo_calc('Real', target_sindarin)
with ut.cd(runfolder):
return _exe(purpose)
else:
raise NotImplementedError
def restore_agent_resources(agents_dir):
ctx.logger.info('Restoring agents in {0}'.format(
utils.AGENTS_ROLLBACK_PATH))
if os.path.isdir(agents_dir):
utils.remove(agents_dir)
utils.mkdir(agents_dir)
utils.copy(os.path.join(utils.AGENTS_ROLLBACK_PATH, 'agents', '.'),
agents_dir)
def run(self, env):
logging.info("Deleting " + str(self.path))
if isinstance(self.path, list) or isinstance(self.path, tuple):
for p in self.path:
utils.remove(p)
else:
utils.remove(self.path)
def restore_agent_resources(agents_dir):
ctx.logger.info('Restoring agents in {0}'.format(
utils.AGENTS_ROLLBACK_PATH))
if os.path.isdir(agents_dir):
utils.remove(agents_dir)
utils.mkdir(agents_dir)
utils.copy(os.path.join(utils.AGENTS_ROLLBACK_PATH, 'agents', '.'),
agents_dir)
def unzip_recording_data(args):
zip_file_path = os.path.join(args.out_dir, 'simpleperf_data.zip')
with zipfile.ZipFile(zip_file_path, 'r') as zip_fh:
names = zip_fh.namelist()
log_info('There are %d recording data files.' % len(names))
for name in names:
log_info('recording file: %s' % os.path.join(args.out_dir, name))
zip_fh.extract(name, args.out_dir)
remove(zip_file_path)
def unzip_recording_data(args):
zip_file_path = os.path.join(args.out_dir, 'simpleperf_data.zip')
with zipfile.ZipFile(zip_file_path, 'r') as zip_fh:
names = zip_fh.namelist()
log_info('There are %d recording data files.' % len(names))
for name in names:
log_info('recording file: %s' % os.path.join(args.out_dir, name))
zip_fh.extract(name, args.out_dir)
remove(zip_file_path)
def install_syncthing():
syncthing_package = \
utils.download_cloudify_resource(
ctx_properties['syncthing_package_url'],
SERVICE_NAME
)
utils.mkdir(HOME_DIR)
utils.untar(syncthing_package, destination=HOME_DIR)
utils.remove(syncthing_package)
def run(self, env):
if not "WORK_DIR" in env:
logging.error("WORK_DIR not defined")
raise TaskError("WORK_DIR not defined")
dir = env["WORK_DIR"]
logging.info("Deleting temporary directory: " + dir)
utils.remove(dir)
logging.info("Successfully deleted temporary directory")
def set_discrete_gpu_state(self, state: bool) -> None:
super().set_discrete_gpu_state(state)
if state:
utils.create_symlink(self.get_config_file('profile.sh'),
profile_file)
self.apply_display_manager_hooks()
else:
utils.remove(profile_file)
self.remove_display_manager_hooks()
def install_syncthing():
syncthing_package = \
utils.download_cloudify_resource(
ctx_properties['syncthing_package_url'],
SERVICE_NAME
)
utils.mkdir(HOME_DIR)
utils.untar(syncthing_package, destination=HOME_DIR)
utils.remove(syncthing_package)
def retrieve_landfire(opt_search, local_path, file_name):
file_path = osp.join(local_path, file_name + '.tif')
info_path = file_path + '.size'
if osp.exists(file_path):
if osp.getsize(file_path) == int(open(info_path).read()):
logging.info('file {} already exists locally'.format(file_path))
return file_path
# submit a job to the Download Service
requestID = server_service('initiateDownload', opt_search)
logging.info('request ID: {}'.format(requestID))
arg = 'downloadID={}'.format(requestID)
stat = None
retries = 50
while stat != 400 and retries:
# call Download service with request id to get status
status = server_service('getDownloadStatus', arg)
logging.info('status: {}'.format(status))
stat = int(status.split(',')[0])
retries -= 1
if not retries and stat != 400:
logging.error(
'maximum number of retries, the server is not responding...')
raise LandfireError('Failed downloading the data...')
# once a status of 400 has been received, retrieve from the URL
url = server_service('getData', arg)
logging.info('download URL: {}'.format(url))
download_url(url, local_path + '.zip')
# send complete message back to server so it can cleanup the job
status = server_service('setDownloadComplete', arg)
logging.info('status: {}'.format(status))
# unzip and save
local_zip = local_path + '.zip'
local_tmp = osp.join(local_path, 'tmp')
with zipfile.ZipFile(local_zip, 'r') as f:
f.extractall(local_tmp)
tif_files = glob.glob(osp.join(local_tmp, '*.tif'))
if len(tif_files) == 1:
tif_file = tif_files[0]
file_size = osp.getsize(tif_file)
move(tif_file, file_path)
with open(ensure_dir(info_path), 'w') as f:
f.write(str(file_size))
else:
logging.warning('not enought or too many TIF files, skipping...')
return None
remove(local_zip)
remove(local_zip + '.size')
delete(local_tmp)
logging.info('data correctly retrieved as {}'.format(file_path))
return file_path
def failedTorrent(self, torrent): url, writer, pathName, callback, args, kwargs = self.activeTorrents.pop(torrent) self.feedStore[url].numFailures += 1 if os.path.exists(pathName): utils.remove(pathName) del self.urlMap[url] if callback: callback(None, *args, **kwargs) if writer: writer.close()
def install_optional(rest_venv):
props = ctx_properties
dsl_parser_source_url = props['dsl_parser_module_source_url']
rest_client_source_url = props['rest_client_module_source_url']
plugins_common_source_url = props['plugins_common_module_source_url']
script_plugin_source_url = props['script_plugin_module_source_url']
agent_source_url = props['agent_module_source_url']
pip_constraints = props['pip_constraints']
rest_service_source_url = props['rest_service_module_source_url']
constraints_file = utils.write_to_tempfile(pip_constraints) \
if pip_constraints else None
# this allows to upgrade modules if necessary.
ctx.logger.info('Installing Optional Packages if supplied...')
if dsl_parser_source_url:
utils.install_python_package(dsl_parser_source_url, rest_venv,
constraints_file)
if rest_client_source_url:
utils.install_python_package(rest_client_source_url, rest_venv,
constraints_file)
if plugins_common_source_url:
utils.install_python_package(plugins_common_source_url, rest_venv,
constraints_file)
if script_plugin_source_url:
utils.install_python_package(script_plugin_source_url, rest_venv,
constraints_file)
if agent_source_url:
utils.install_python_package(agent_source_url, rest_venv,
constraints_file)
if rest_service_source_url:
ctx.logger.info('Downloading cloudify-manager Repository...')
manager_repo = \
utils.download_cloudify_resource(rest_service_source_url,
SERVICE_NAME)
ctx.logger.info('Extracting Manager Repository...')
tmp_dir = utils.untar(manager_repo, unique_tmp_dir=True)
rest_service_dir = join(tmp_dir, 'rest-service')
resources_dir = join(tmp_dir, 'resources/rest-service/cloudify/')
ctx.logger.info('Installing REST Service...')
utils.install_python_package(rest_service_dir, rest_venv,
constraints_file)
ctx.logger.info('Deploying Required Manager Resources...')
utils.move(resources_dir, utils.MANAGER_RESOURCES_HOME)
utils.remove(tmp_dir)
if constraints_file:
os.remove(constraints_file)
def delete(self, topic_id):
if topic_id not in self.data:
return
for tid in self.data[topic_id]['recommendations']:
remove(self.target_corpus.data[tid]['appears_in_special'],
topic_id)
del self.data[topic_id]
self.logger.info('Topic %s deleted', topic_id)
def test_read_config(self):
temp_stdout(self)
with open(INI, 'w') as f:
f.write(INI_CONF)
sys.argv[1:] = ["-r", INI]
parser.add_argument("-a", "--arg1")
parser.add_argument("-b", "--arg2", action="store_true")
initialize(add_config=True)
self.assertEqual(args.arg1, "test")
self.assertTrue(args.arg2)
remove(INI)
def test_write_config(self):
sys.argv[1:] = ["--arg1", "test", "--arg2", "-w", INI]
parser.add_argument("-a", "--arg1")
parser.add_argument("-b", "--arg2", action="store_true")
initialize(add_config=True)
_save_config(globals())
with open(INI) as f:
ini = f.read().strip()
self.assertIn("arg1", ini)
self.assertIn("arg2", ini)
remove(INI)
def trac(content, theme, resource_fetcher, log):
theme('#sidebar').empty()
utils.remove(content, 'br')
nav = pq('<h2>Trac</h2><ul class="subnav"></ul>')
menu = nav('ul')
items = content('#mainnav li')
for i in items:
i = pq(i)
i.attr(class_='__no_css')
menu.append(i)
theme('#sidebar').append(nav)
def main() -> None:
"""Main function that is call when the script is run."""
utils.rename(OUTPUT_FILE, utils.get_date())
driver = set_scraper()
site_login(driver)
get_mail_link(driver)
scrape_mail(driver)
save_result(driver)
site_logout(driver)
utils.remove(PAGE_URLS)
print("Scraping has finished successfully.")
def createCustomerRelease(releaseType, releaseSpecDir, destDir, fromDir, releaseName, allArchitectures=True, allArchiveTypes=False, allowSymbolicLinks=False, parentTmpDir=None):
"""Create a customer release tarball in destdir from the
directories engReleaseDir and releaseSpecDir
releaseType is one of biaryReleaseTypes or sourceReleaseTypes
fromDir is engineering build for the binary release,
source dir for a source release
releaseSpecDir is directory containing manifests
Returns a path to the tarball
Throws an exception on error"""
if not os.path.exists(destDir):
utils.createDir(destDir)
if releaseType in binaryReleaseTypes:
isSourceRelease = False
elif releaseType in sourceReleaseTypes:
isSourceRelease = True
else:
raise Exception("Unknown release type '%s'" % releaseType)
if not isSourceRelease:
allArchitectures=False
manifestFile = os.path.join(releaseSpecDir, releaseType + "_release",
"manifests", releaseType + "_release.manifest")
log.info("Creating customer release of type %s", releaseType)
log.info("specDir: %s", releaseSpecDir)
log.info("fromDir: %s", fromDir)
tempdir = utils.createTemporaryDirectory("assemble_release", parentDir=parentTmpDir)
try:
releaseImageDir = os.path.join(tempdir, releaseName)
installFromManifest(manifestFile, fromDir, releaseImageDir,
level=0, overwrite=False, destdirExists=False,
allArchitectures=allArchitectures, allowSymbolicLinks=allowSymbolicLinks)
if isSourceRelease:
# run autogen equivalent on source trees
build.prepareSourceTree(releaseImageDir, customerRelease=True)
# Need to create both zip and tgz source releases on the master build system
if allArchiveTypes == True:
# sanity checks
if not isSourceRelease:
raise Exception("Attempt to create both zip and tar file for a binary release. Probably an error")
if getArch() == "win32":
raise Exception("Attempt to create .tgz source release on a windows system. Would not include autogen.sh")
zipFile = utils.createArchive(releaseImageDir, destDir, type="zip")
tarFile = utils.createArchive(releaseImageDir, destDir, type="tar")
return (tarFile, zipFile)
else:
release = utils.createArchive(releaseImageDir, destDir)
return release
finally:
utils.remove(tempdir)
def delete(self, topic_id):
if topic_id not in self.data:
return
for tid in self.data[topic_id][
'appears_in']: # list of topic id's whose similarity lists tid appears in
if tid in self.data:
remove(self.data[tid]['sim_list'], topic_id)
self.data[tid]['updated'] = True
del self.data[topic_id]
self.logger.info('Topic %s deleted (%d)', topic_id, len(self.data))
def execute_before_bootstrap():
exec_paths = ctx_properties['execute_before_bootstrap']
for path in exec_paths:
delete_tmp_path = False
# TODO: Upon moving to Python 3, convert to urllib2.urlparse
if '://' in path and path.split('://', 1)[0] in ('http', 'https'):
path = utils.download_file(path)
utils.chmod('744', path)
delete_tmp_path = True
utils.run(path)
if delete_tmp_path:
utils.remove(path)
def test_common_utility_functions(self):
self.assertEqual(xor("this is a test", " "), "THIS\x00IS\x00A\x00TEST")
self.assertEqual(list(strings("this is a \x00 test")), ["this is a ", " test"])
FILE, CONTENT = ".test_strings", b"this is a \x00 test"
with open(FILE, 'wb') as f:
f.write(CONTENT)
self.assertIsNone(xor_file(FILE, " "))
self.assertIsNone(xor_file(FILE, " "))
with open(FILE, 'rb') as f:
self.assertEqual(f.read(), CONTENT)
self.assertEqual(list(strings_from_file(FILE)), ["this is a ", " test"])
remove(FILE)
def _install_stage():
nodejs_source_url = ctx_properties['nodejs_tar_source_url']
stage_source_url = ctx_properties['stage_tar_source_url']
if not utils.resource_factory.local_resource_exists(stage_source_url):
ctx.logger.info('Stage package not found in manager resources '
'package. Stage will not be installed.')
ctx.instance.runtime_properties['skip_installation'] = 'true'
return
# injected as an input to the script
ctx.instance.runtime_properties['influxdb_endpoint_ip'] = \
os.environ.get('INFLUXDB_ENDPOINT_IP')
utils.set_selinux_permissive()
utils.copy_notice(SERVICE_NAME)
utils.mkdir(NODEJS_DIR)
utils.mkdir(HOME_DIR)
utils.mkdir(LOG_DIR)
utils.create_service_user(STAGE_USER, STAGE_GROUP, HOME_DIR)
ctx.logger.info('Installing NodeJS...')
nodejs = utils.download_cloudify_resource(nodejs_source_url, SERVICE_NAME)
utils.untar(nodejs, NODEJS_DIR)
utils.remove(nodejs)
ctx.logger.info('Installing Cloudify Stage (UI)...')
stage_tar = utils.download_cloudify_resource(stage_source_url,
SERVICE_NAME)
utils.untar(stage_tar, HOME_DIR)
utils.remove(stage_tar)
ctx.logger.info('Fixing permissions...')
utils.chown(STAGE_USER, STAGE_GROUP, HOME_DIR)
utils.chown(STAGE_USER, STAGE_GROUP, NODEJS_DIR)
utils.chown(STAGE_USER, STAGE_GROUP, LOG_DIR)
utils.deploy_sudo_command_script(
'restore-snapshot.py',
'Restore stage directories from a snapshot path',
component=SERVICE_NAME,
allow_as=STAGE_USER)
utils.chmod('a+rx', '/opt/cloudify/stage/restore-snapshot.py')
utils.logrotate(SERVICE_NAME)
utils.systemd.configure(SERVICE_NAME)
backend_dir = join(HOME_DIR, 'backend')
npm_path = join(NODEJS_DIR, 'bin', 'npm')
subprocess.check_call(
'cd {0}; {1} run db-migrate'.format(backend_dir, npm_path),
shell=True)
def wrapper_is_compat(l1, l2):
# Make l1 and l2 Items
i1 = Items(l1)
i2 = Items(l2)
remove('test.h5.tmp')
with h5py.File('test.h5.tmp') as h5file:
g = h5file.create_group('deleteme')
i2.create_dataset(g, 10)
i2.write(g)
res = i1.is_appendable_to(g)
remove('test.h5.tmp')
return res
def post_processing(self):
cw = self.cw
with cw.convert(self):
outdir = get_outdir(self.type)
out = os.path.join(outdir, self.title + '.mp4')
ffmpeg.join(cw.names, out, cw)
for file in cw.names:
utils.remove(file)
cw.setNameAt(0, out)
del cw.imgs[1:]
cw.dones.add(os.path.realpath(out))
cw.dir = outdir
def wiki(content, theme, resource_fetcher, log):
header = content('#header')
utils.remove(content, '#username', '#logo', '#searchform', '#footer')
nav = pq('<h2>Wiki</h2><ul class="subnav"></ul>')
menu = nav('ul')
items = header('li')
for i in items:
menu.append(pq(i))
theme('#sidebar').append(nav)
header.remove()
for i in content('img'):
i = pq(i)
i.attr.src = i.attr.src.replace(':8096', '')
def collect_native_libs_on_device(self):
self.device_build_id_map.clear()
self.adb.check_run(['shell', 'mkdir', '-p', self.dir_on_device])
if os.path.exists(self.build_id_list_file):
os.remove(self.build_id_list_file)
self.adb.run(['pull', self.dir_on_device + self.build_id_list_file])
if os.path.exists(self.build_id_list_file):
with open(self.build_id_list_file, 'rb') as fh:
for line in fh.readlines():
line = bytes_to_str(line).strip()
items = line.split('=')
if len(items) == 2:
self.device_build_id_map[items[0]] = items[1]
remove(self.build_id_list_file)
def extractCanvasSettings(d):
"""Split a dict in canvas settings and other items.
Returns a tuple of two dicts: the first one contains the items
that are canvas settings, the second one the rest.
"""
return utils.select(d,pf.refcfg['canvas']),utils.remove(d,pf.refcfg['canvas'])
def calculate_part(lexeme, form, parse):
if parse[2] == "N":
stem_key, results = getattr(lexeme, "rev_" + parse)(form)
else:
tvm, pn = parse.split(".")
stem_key, results = getattr(lexeme, "rev_" + tvm)(form, pn)
return stem_key, set(remove(result) for result in results)
def _handle_file(self, vals, res=[]):
src = vals.pop()
_vals = vals
df_vals = []
res.append(src)
for val in _vals:
if is_diff(src, val):
print 'Not diff: {0} , {1}'.format(src, val)
df_vals.append(val)
continue
else:
remove(val)
if len(df_vals) > 1:
self._handle_file(df_vals, res)
else:
res = res + df_vals
return res
def install_consul():
consul_binary = join(HOME_DIR, 'consul')
utils.mkdir(dirname(consul_binary))
utils.mkdir(CONFIG_DIR)
consul_package = \
utils.download_cloudify_resource(ctx_properties['consul_package_url'],
SERVICE_NAME)
temp_dir = tempfile.mkdtemp()
try:
with zipfile.ZipFile(consul_package) as consul_archive:
consul_archive.extractall(temp_dir)
utils.move(join(temp_dir, 'consul'), consul_binary)
utils.chmod('+x', consul_binary)
finally:
utils.remove(temp_dir)
def reverse(form, pn=None):
stems = set()
if pn:
if pn in paradigm:
endings = paradigm[pn]
else:
endings = []
else:
endings = paradigm
for ending, stem_ending in endings:
if remove(form).endswith(remove(remove_length(ending))):
stems.add(form[:-len(ending)] + (stem_ending if stem_ending else ""))
if stems:
return stems
else:
raise Exception("got a {} of form {}".format(pn, form))
def install_consul():
consul_binary = '/opt/cloudify/consul/consul'
consul_config_dir = '/etc/consul.d'
utils.mkdir(dirname(consul_binary))
utils.mkdir(consul_config_dir)
consul_package = \
utils.download_cloudify_resource(ctx_properties['consul_package_url'],
CONSUL_SERVICE_NAME)
temp_dir = tempfile.mkdtemp()
try:
with zipfile.ZipFile(consul_package) as consul_archive:
consul_archive.extractall(temp_dir)
utils.move(join(temp_dir, 'consul'), consul_binary)
utils.chmod('+x', consul_binary)
finally:
utils.remove(temp_dir)
def postbuild_win32(srcdir, installdir):
srcdir = os.path.abspath(os.path.normpath(os.path.expanduser(srcdir)))
installdir = os.path.abspath(os.path.normpath(os.path.expanduser(installdir)))
# Delete .exp, .pdb and .lib files
extensionsToDelete = [".exp", ".pdb", ".lib"]
log.info("Performing postbuild processing for win32. Deleting: %s", extensionsToDelete)
for (root, dirs, files) in os.walk(installdir):
# don't delete the precompiled runtime and node libraries
for exclude in ['release', 'debug']:
if exclude in dirs:
dirs.remove(exclude)
# this file is needed for linking!
# todo: remove the special case
excludedBases = ['nta_vision']
for f in files:
(base, ext) = os.path.splitext(f)
if ext in extensionsToDelete and base not in excludedBases:
path = os.path.join(root, f)
utils.remove(path)
log.info("Postbuild complete")
def install_optional(rest_venv):
props = ctx_properties
dsl_parser_source_url = props['dsl_parser_module_source_url']
rest_client_source_url = props['rest_client_module_source_url']
plugins_common_source_url = props['plugins_common_module_source_url']
script_plugin_source_url = props['script_plugin_module_source_url']
agent_source_url = props['agent_module_source_url']
rest_service_source_url = props['rest_service_module_source_url']
# this allows to upgrade modules if necessary.
ctx.logger.info('Installing Optional Packages if supplied...')
if dsl_parser_source_url:
utils.install_python_package(dsl_parser_source_url, rest_venv)
if rest_client_source_url:
utils.install_python_package(rest_client_source_url, rest_venv)
if plugins_common_source_url:
utils.install_python_package(plugins_common_source_url, rest_venv)
if script_plugin_source_url:
utils.install_python_package(script_plugin_source_url, rest_venv)
if agent_source_url:
utils.install_python_package(agent_source_url, rest_venv)
if rest_service_source_url:
ctx.logger.info('Downloading cloudify-manager Repository...')
manager_repo = \
utils.download_cloudify_resource(rest_service_source_url,
SERVICE_NAME)
ctx.logger.info('Extracting Manager Repository...')
tmp_dir = utils.untar(manager_repo, unique_tmp_dir=True)
rest_service_dir = join(tmp_dir, 'rest-service')
resources_dir = join(tmp_dir, 'resources/rest-service/cloudify/')
ctx.logger.info('Installing REST Service...')
utils.install_python_package(rest_service_dir, rest_venv)
ctx.logger.info('Deploying Required Manager Resources...')
utils.move(resources_dir, utils.MANAGER_RESOURCES_HOME)
utils.remove(tmp_dir)
def forward(stems, pn=None):
forms = []
for stem in stems:
if pn:
pp = paradigm[pn]
else:
pp = paradigm
for ending, stem_ending in pp:
stem2, stem_ending = (stem[:-len(stem_ending)], remove(stem_ending)) if stem_ending else (stem, "")
if stem.endswith(stem_ending):
forms.append(stem2 + ending)
return forms
def get_manager_config():
"""
Extracting specific files from cloudify-manager repo, with clean-ups after
"""
cloudify_resources_url = ctx_properties['cloudify_resources_url']
ctx.logger.info('Downloading cloudify-manager Repository...')
manager_repo = utils.download_cloudify_resource(
cloudify_resources_url, SERVICE_NAME)
ctx.logger.info('Extracting Manager Repository...')
manager_dir = utils.untar(manager_repo, unique_tmp_dir=True)
ctx.logger.info('Deploying Riemann manager.config...')
config_src_path = join(
manager_dir, 'plugins', 'riemann-controller',
'riemann_controller', 'resources', 'manager.config'
)
utils.move(
config_src_path,
'{0}/conf.d/manager.config'.format(RIEMANN_CONFIG_PATH)
)
utils.remove(manager_dir)
utils.remove(manager_repo)
def _install_optional(mgmtworker_venv):
rest_props = utils.ctx_factory.get('restservice')
rest_client_source_url = rest_props['rest_client_module_source_url']
plugins_common_source_url = rest_props['plugins_common_module_source_url']
script_plugin_source_url = rest_props['script_plugin_module_source_url']
rest_service_source_url = rest_props['rest_service_module_source_url']
agent_source_url = rest_props['agent_module_source_url']
# this allows to upgrade modules if necessary.
ctx.logger.info('Installing Optional Packages if supplied...')
if rest_client_source_url:
utils.install_python_package(rest_client_source_url, mgmtworker_venv)
if plugins_common_source_url:
utils.install_python_package(
plugins_common_source_url, mgmtworker_venv)
if script_plugin_source_url:
utils.install_python_package(script_plugin_source_url, mgmtworker_venv)
if agent_source_url:
utils.install_python_package(agent_source_url, mgmtworker_venv)
if rest_service_source_url:
ctx.logger.info('Downloading cloudify-manager Repository...')
manager_repo = \
utils.download_cloudify_resource(rest_service_source_url,
SERVICE_NAME)
ctx.logger.info('Extracting Manager Repository...')
tmp_dir = utils.untar(manager_repo, unique_tmp_dir=True)
workflows_dir = join(tmp_dir, 'workflows')
riemann_dir = join(tmp_dir, 'plugins/riemann-controller')
ctx.logger.info('Installing Management Worker Plugins...')
utils.install_python_package(riemann_dir, mgmtworker_venv)
utils.install_python_package(workflows_dir, mgmtworker_venv)
utils.remove(tmp_dir)
def removeimage(self, typ, basedir="", delete=True):
""" Method to remove a specific image from the class, including from
disk if requested.
Parameters
----------
type : string
The type of the image to be removed. Can be any of the
following: bt.AUX to remove the auxiliary file,
bt.THUMB to remove the thumbnail, or a file format
(e.g. bt.FITS) to remove the requested format from
the main image dictionary.
delete : Boolean
Whether to delete the actual image from disk (default is True)
Returns
-------
None
"""
# go through each image component and delete the file and instance
if(typ == bt.THUMB):
if(delete):
utils.remove(basedir + os.sep + self.thumbnail)
self.thumbnail = ""
self.thumbnailtype = ""
elif(typ == bt.AUX):
if(delete):
utils.remove(basedir + os.sep+ self.auxiliary)
self.auxiliary = ""
self.auxtype = ""
elif(typ in bt.image_types and typ in self.images):
if(delete):
utils.remove(basedir + os.sep + self.images[typ])
elif(typ in self.images.values()):
k = None
for key, val in self.images.iteritems():
if(typ == val):
k = key
if(delete) :
utils.remove(basedir + os.sep + k)
self.images.pop(k, None)
else:
pass
def perform_sanity():
ctx.logger.info('Starting Manager sanity check...')
_prepare_sanity_app()
ctx.logger.info('Installing sanity app...')
exec_id = _install_sanity_app()
ctx.logger.info('Sanity app installed. Performing sanity test...')
_assert_webserver_running()
_assert_logs_and_events(exec_id)
ctx.logger.info('Manager sanity check successful, '
'cleaning up sanity resources.')
_cleanup_sanity()
# the 'run_sanity' parameter is injected explicitly from the cli as an
# operation parameter with 'true' as its value.
# This is done to prevent the sanity test from running before the
# provider context is available.
if os.environ.get('run_sanity') == 'true' or \
utils.is_upgrade or \
utils.is_rollback:
perform_sanity()
if utils.is_upgrade or utils.is_rollback:
utils.restore_upgrade_snapshot()
if utils.is_upgrade:
utils.set_upgrade_success_in_upgrade_meta()
if utils.is_rollback:
# remove data created by the upgrade process.
utils.remove(utils.UPGRADE_METADATA_FILE)