def setUp(self):
self.cpcode_path = NS_CPCODE
self.temp_ns_dir = "/{0}/{1}".format(self.cpcode_path, str(uuid.uuid4()))
self.temp_file = "{0}.txt".format(str(uuid.uuid4()))
self.temp_ns_file = "{0}/{1}".format(self.temp_ns_dir, self.temp_file)
self.ns = Netstorage(NS_HOSTNAME, NS_KEYNAME, NS_KEY)
def uploadToNS():
# Fetch s3 Variables from Env from AWS
S3_Bucket = os.environ['S3_Bucket']
S3_Path = os.environ['S3_PATH']
S3_File = os.environ['S3_File']
#Fetch Akamai NS Variables from Env from AWS
NS_HOSTNAME = os.environ['NS_HOSTNAME']
NS_KEYNAME = os.environ['NS_KEYNAME']
NS_KEY = os.environ['NS_KEY']
getObjectFromS3(S3_Bucket,S3_Path,S3_File)
ns = Netstorage(NS_HOSTNAME, NS_KEYNAME, NS_KEY, ssl=False)
if os.path.exists('/tmp/'+S3_File):
ok, response = ns.upload('/tmp/'+S3_File,'/'+S3_Path+str(NS_CP)+'/')
else:
return {
'status':"ERROR",
'body': "The File /tmp/"+S3_File+" Does Not Exist!"
}
if ok:
if os.path.exists('/tmp/'+S3_File):
os.remove('/tmp/'+S3_File)
return {
'status':response.status_code,
'body': json.dumps(response.text)
}
def upload(CPCODE, path):
"""
Description: uploads to netstorage.
Links:
https://learn.akamai.com/en-us/webhelp/netstorage/netstorage-http-api-developer-guide/GUID-C9CEE090-B272-4E47-ACD9-853D80C747BC.html
https://github.com/akamai/NetStorageKit-Python
Expects:
Cpcode
path: s3 object key
Returns: Bool based on task success.
"""
NS_USER = json.loads(SECRET)['NS_USER']
NS_KEY = json.loads(SECRET)['NS_KEY']
NS_HOSTNAME = json.loads(SECRET)['NS_HOSTNAME']
logger.info("Uploading to Ns with user: '******'".format(NS_USER))
ns = Netstorage(NS_HOSTNAME, NS_USER, NS_KEY, ssl=False)
netstorage_destination = "/{0}/{1}".format(CPCODE, path)
logger.info("Upload Location: '{0}'.".format(netstorage_destination))
ok, _ = ns.upload(path, netstorage_destination)
if ok:
logger.info("File: uploaded.")
return True
else:
logger.info("File: upload failed.")
return False
def create_netstorage_paths(sformat, geo, cpcodes, connectionDetails):
'''Connects to Akamai's NetStorage infrastructure and creates file
paths to receive log delivery via FTP
Keyword Arguments:
sformat -- string provided by user indicating streaming format
geo -- string provided by user indicating service geo
cpcodes -- list of cpcodes provided by user
connectionDetails -- ConfigParser object containing NS auth details
'''
for cpcode in cpcodes:
ns = Netstorage(connectionDetails['DEFAULT']['Hostname'],
connectionDetails['DEFAULT']['Key-name'],
connectionDetails['DEFAULT']['Key'],
ssl=True)
ns_dir = '/{0}/logdelivery/{1}/{2}/{3}'\
.format(connectionDetails['DEFAULT']['Cpcode'], sformat, geo, cpcode)
print('Creating {0}'.format(ns_dir), flush=True)
ok, response = ns.mkdir(ns_dir)
if ok is True:
print('Sucessfully created path {0}.'.format(ns_dir), flush=True)
else:
print('Error creating {0}. The error response code was {1}: {2}'.
format(ns_dir, response.status_code, response.text),
flush=True)
def setUp(self):
self.cpcode_path = NS_CPCODE
self.temp_ns_dir = "/{0}/{1}".format(self.cpcode_path, str(uuid.uuid4()))
self.temp_file = "{0}.txt".format(str(uuid.uuid4()))
self.temp_ns_file = "{0}/{1}".format(self.temp_ns_dir, self.temp_file)
self.ns = Netstorage(NS_HOSTNAME, NS_KEYNAME, NS_KEY)
def test_create_netstorage_paths():
sformat = 'smooth'
geo = 'us'
cpcodes = ['123456', '789012']
connectionDetails = alda.get_netstorage_credentials('alda.netstorage')
ns = Netstorage(connectionDetails['DEFAULT']['Hostname'],
connectionDetails['DEFAULT']['Key-name'],
connectionDetails['DEFAULT']['Key'], ssl=True)
for cpcode in cpcodes:
ns_dir = '/{0}/logdelivery/{1}/{2}/{3}'\
.format(connectionDetails['DEFAULT']['Cpcode'], sformat, geo, cpcode)
ok, response = ns.mkdir(ns_dir) # Test we can make directories
assert ok is True
assert response.status_code == 200
ok, response = ns.rmdir(ns_dir) # Remove test directories
print(ok)
print(response.text)
assert ok is True
assert response.status_code == 200
def removeObject(path):
"""
Description: Remove objects from NS
Links:
https://learn.akamai.com/en-us/webhelp/netstorage/netstorage-http-api-developer-guide/GUID-C9CEE090-B272-4E47-ACD9-853D80C747BC.html
https://github.com/akamai/NetStorageKit-Python
Expects:
path: Path to be deleted in NS.
Returns: Bool based on task success.
"""
NS_USER = json.loads(NS_SECRET)['NS_USER']
NS_KEY = json.loads(NS_SECRET)['NS_KEY']
NS_HOSTNAME = json.loads(NS_SECRET)['NS_HOSTNAME']
ns = Netstorage(NS_HOSTNAME, NS_USER, NS_KEY, ssl=False)
netstorage_destination = "/{0}/{1}".format(CPCODE, path)
ok, response = ns.delete(netstorage_destination)
if ok:
logger.info("File {0} deleted.".format(path))
else:
logger.error("File {0} not deleted: {1}".format(
path, response.status_code))
return response.status_code
return response.status_code
from akamai.netstorage import Netstorage, NetstorageError
import xml.etree.ElementTree as ET
NS_HOSTNAME = 'samsung_flagship_image_imc_eu-nsu.akamaihd.net'
NS_KEYNAME = 'jamespark_IMC_EU'
NS_KEY = 'lP1WSEN2C6xUg64u0aifgWtxrIqdU60WB4X6ff0l5fo2qq8x4t'
NS_CPCODE = '606569'
ns = Netstorage(NS_HOSTNAME, NS_KEYNAME, NS_KEY, ssl=False)
info = {}
def recursive(items):
for item in items:
if item.get('type') == 'file':
k = items.attrib['directory'][7:]
if k not in info.keys():
info[k] = [1, int(item.get('size'))]
else:
info[k][0] += 1
info[k][1] += int(item.get('size'))
elif item.get('type') == 'dir':
ok, res = ns.dir(items.attrib['directory'] + "/" +
item.get('name'))
if ok:
xml_tree = ET.fromstring(res.content)
try:
recursive(xml_tree)
except:
pass
'--keyname',
dest='keyname',
help='Netstorage API keyname ex) xxxxx')
parser.add_option('-K',
'--key',
dest='key',
help='Netstorage API key ex) xxxxxxxxxxxxx')
parser.add_option('-a', '--action', dest='action')
parser.add_option('-s', '--ssl', dest='ssl')
parser.add_option('-t', '--timeout', dest='timeout')
(options, args) = parser.parse_args()
if options.hostname and options.keyname and options.key and options.action:
ssl = options.ssl if hasattr(options, 'ssl') else False
ns = Netstorage(options.hostname, options.keyname, options.key, ssl)
def _arg(key, default=None):
return args[key] if key in args else default
try:
res = None
if options.action == 'delete':
ok, res = ns.delete(args[0], _arg(1))
elif options.action == 'dir':
if len(args) >= 2:
ok, res = ns.dir(args[0], ast.literal_eval(args[1]),
_arg(2))
else:
ok, res = ns.dir(args[0])
elif options.action == 'list':
class TestNetstorage(unittest.TestCase):
def setUp(self):
self.cpcode_path = NS_CPCODE
self.temp_ns_dir = "/{0}/{1}".format(self.cpcode_path, str(uuid.uuid4()))
self.temp_file = "{0}.txt".format(str(uuid.uuid4()))
self.temp_ns_file = "{0}/{1}".format(self.temp_ns_dir, self.temp_file)
self.ns = Netstorage(NS_HOSTNAME, NS_KEYNAME, NS_KEY)
def tearDown(self):
# delete temp files for local
if os.path.exists(self.temp_file):
os.remove(self.temp_file)
print("[TEARDOWN] remove {0} from local done".format(self.temp_file))
if os.path.exists(self.temp_file + '_rename'):
os.remove(self.temp_file + '_rename')
print("[TEARDOWN] remove {0} from local done".format(self.temp_file + '_rename'))
# delete temp files for netstorage
ok, _ = self.ns.delete(self.temp_ns_file)
if ok:
print("[TEARDOWN] delete {0} done".format(self.temp_ns_file))
ok, _ = self.ns.delete(self.temp_ns_file + '_lnk')
if ok:
print("[TEARDOWN] delete {0} done".format(self.temp_ns_file + '_lnk'))
ok, _ = self.ns.delete(self.temp_ns_file + '_rename')
if ok:
print("[TEARDOWN] delete {0} done".format(self.temp_ns_file + '_rename'))
ok, _ = self.ns.rmdir(self.temp_ns_dir)
if ok:
print("[TEARDOWN] rmdir {0} done".format(self.temp_ns_dir))
def test_netstorage(self):
# dir
ok, _ = self.ns.dir("/" + self.cpcode_path)
self.assertEqual(True, ok, "dir fail.")
print("[TEST] dir {0} done".format("/" + self.cpcode_path))
# mkdir
ok, _ = self.ns.mkdir(self.temp_ns_dir)
self.assertEqual(True, ok, "mkdir fail.")
print("[TEST] mkdir {0} done".format(self.temp_ns_dir))
# upload
with open(self.temp_file, 'wt') as f:
f.write("Hello, Netstorage API World!")
ok, res = self.ns.upload(self.temp_file, self.temp_ns_file)
self.assertEqual(True, ok, "upload fail.")
print("[TEST] upload {0} to {1} done".format(self.temp_file, self.temp_ns_file))
# du
ok, res = self.ns.du(self.temp_ns_dir)
self.assertEqual(True, ok)
xml_tree = ET.fromstring(res.content)
self.assertEqual(str(os.stat(self.temp_file).st_size), xml_tree[0].get('bytes'), "du fail.")
print("[TEST] du done")
# mtime
current_time = int(time.time())
ok, _ = self.ns.mtime(self.temp_ns_file, current_time)
self.assertEqual(True, ok, "mtime fail.")
print("[TEST] mtime {0} to {1} done".format(self.temp_ns_file, current_time))
# stat
ok, res = self.ns.stat(self.temp_ns_file)
self.assertEqual(True, ok, "stat fail.")
xml_tree = ET.fromstring(res.text)
self.assertEqual(str(current_time), xml_tree[0].get('mtime'))
print("[TEST] stat done")
# symlink
ok, _ = self.ns.symlink(self.temp_ns_file, self.temp_ns_file + "_lnk")
self.assertEqual(True, ok, "symlink fail.")
print("[TEST] symlink {0} to {1} done".format(self.temp_ns_file, self.temp_ns_file + "_lnk"))
# rename
ok, _ = self.ns.rename(self.temp_ns_file, self.temp_ns_file + "_rename")
self.assertEqual(True, ok, "rename fail.")
print("[TEST] rename {0} to {1} done".format(self.temp_ns_file, self.temp_ns_file + "_rename"))
# download
ok, _ = self.ns.download(self.temp_ns_file + "_rename")
self.assertEqual(True, ok, "download fail.")
print("[TEST] download {0} done".format(self.temp_ns_file + "_rename"))
# delete
ok, _ = self.ns.delete(self.temp_ns_file + "_rename")
self.assertEqual(True, ok, "delete fail.")
print("[TEST] delete {0} done".format(self.temp_ns_file + "_rename"))
ok, _ = self.ns.delete(self.temp_ns_file + "_lnk")
self.assertEqual(True, ok, "delete fail.")
print("[TEST] delete {0} done".format(self.temp_ns_file + "_lnk"))
# rmdir
ok, _ = self.ns.rmdir(self.temp_ns_dir)
self.assertEqual(True, ok, "rmdir fail.")
print("[TEST] rmdir {0} done".format(self.temp_ns_dir))
def test_netstorage_exception(self):
print(os.linesep)
with self.assertRaises(NetstorageError):
self.ns.dir("Invalid ns path")
print("[TEST] Invalid ns path NetstorageError test done")
with self.assertRaises(NetstorageError):
self.ns.upload("Invalid local path", self.temp_ns_file)
print("[TEST] Invalid local path NetstorageError test done")
with self.assertRaises(NetstorageError):
self.ns.download("/123456/directory/", self.temp_file)
print("[TEST] Download directory path NetstorageError test done")
class TestNetstorage(unittest.TestCase):
def setUp(self):
self.cpcode_path = NS_CPCODE
self.temp_ns_dir = "/{0}/{1}".format(self.cpcode_path, str(uuid.uuid4()))
self.temp_file = "{0}.txt".format(str(uuid.uuid4()))
self.temp_ns_file = "{0}/{1}".format(self.temp_ns_dir, self.temp_file)
self.ns = Netstorage(NS_HOSTNAME, NS_KEYNAME, NS_KEY)
def tearDown(self):
# delete temp files for local
if os.path.exists(self.temp_file):
os.remove(self.temp_file)
print("[TEARDOWN] remove {0} from local done".format(self.temp_file))
if os.path.exists(self.temp_file + '_rename'):
os.remove(self.temp_file + '_rename')
print("[TEARDOWN] remove {0} from local done".format(self.temp_file + '_rename'))
# delete temp files for netstorage
ok, _ = self.ns.delete(self.temp_ns_file)
if ok:
print("[TEARDOWN] delete {0} done".format(self.temp_ns_file))
ok, _ = self.ns.delete(self.temp_ns_file + '_lnk')
if ok:
print("[TEARDOWN] delete {0} done".format(self.temp_ns_file + '_lnk'))
ok, _ = self.ns.delete(self.temp_ns_file + '_rename')
if ok:
print("[TEARDOWN] delete {0} done".format(self.temp_ns_file + '_rename'))
ok, _ = self.ns.rmdir(self.temp_ns_dir)
if ok:
print("[TEARDOWN] rmdir {0} done".format(self.temp_ns_dir))
def test_netstorage(self):
# dir
ok, _ = self.ns.dir("/" + self.cpcode_path)
self.assertEqual(True, ok, "dir fail.")
print("[TEST] dir {0} done".format("/" + self.cpcode_path))
# mkdir
ok, _ = self.ns.mkdir(self.temp_ns_dir)
self.assertEqual(True, ok, "mkdir fail.")
print("[TEST] mkdir {0} done".format(self.temp_ns_dir))
# upload
with open(self.temp_file, 'wt') as f:
f.write("Hello, Netstorage API World!")
ok, res = self.ns.upload(self.temp_file, self.temp_ns_file)
self.assertEqual(True, ok, "upload fail.")
print("[TEST] upload {0} to {1} done".format(self.temp_file, self.temp_ns_file))
# du
ok, res = self.ns.du(self.temp_ns_dir)
self.assertEqual(True, ok)
xml_tree = ET.fromstring(res.content)
self.assertEqual(str(os.stat(self.temp_file).st_size), xml_tree[0].get('bytes'), "du fail.")
print("[TEST] du done")
# mtime
current_time = int(time.time())
ok, _ = self.ns.mtime(self.temp_ns_file, current_time)
self.assertEqual(True, ok, "mtime fail.")
print("[TEST] mtime {0} to {1} done".format(self.temp_ns_file, current_time))
# stat
ok, res = self.ns.stat(self.temp_ns_file)
self.assertEqual(True, ok, "stat fail.")
xml_tree = ET.fromstring(res.text)
self.assertEqual(str(current_time), xml_tree[0].get('mtime'))
print("[TEST] stat done")
# symlink
ok, _ = self.ns.symlink(self.temp_ns_file, self.temp_ns_file + "_lnk")
self.assertEqual(True, ok, "symlink fail.")
print("[TEST] symlink {0} to {1} done".format(self.temp_ns_file, self.temp_ns_file + "_lnk"))
# rename
ok, _ = self.ns.rename(self.temp_ns_file, self.temp_ns_file + "_rename")
self.assertEqual(True, ok, "rename fail.")
print("[TEST] rename {0} to {1} done".format(self.temp_ns_file, self.temp_ns_file + "_rename"))
# download
ok, _ = self.ns.download(self.temp_ns_file + "_rename")
self.assertEqual(True, ok, "download fail.")
print("[TEST] download {0} done".format(self.temp_ns_file + "_rename"))
# delete
ok, _ = self.ns.delete(self.temp_ns_file + "_rename")
self.assertEqual(True, ok, "delete fail.")
print("[TEST] delete {0} done".format(self.temp_ns_file + "_rename"))
ok, _ = self.ns.delete(self.temp_ns_file + "_lnk")
self.assertEqual(True, ok, "delete fail.")
print("[TEST] delete {0} done".format(self.temp_ns_file + "_lnk"))
# rmdir
ok, _ = self.ns.rmdir(self.temp_ns_dir)
self.assertEqual(True, ok, "rmdir fail.")
print("[TEST] rmdir {0} done".format(self.temp_ns_dir))
def test_netstorage_exception(self):
print(os.linesep)
if not (sys.version_info[0] == 2 and sys.version_info[1] <= 6):
with self.assertRaises(NetstorageError):
self.ns.dir("Invalid ns path")
print("[TEST] Invalid ns path NetstorageError test done")
with self.assertRaises(NetstorageError):
self.ns.upload("Invalid local path", self.temp_ns_file)
print("[TEST] Invalid local path NetstorageError test done")
with self.assertRaises(NetstorageError):
self.ns.download("/123456/directory/", self.temp_file)
print("[TEST] Download directory path NetstorageError test done")
from akamai.netstorage import Netstorage, NetstorageError
import xml.etree.ElementTree as ET
NS_HOSTNAME = 'hostname'
NS_KEYNAME = 'keyname'
NS_KEY = 'key'
NS_CPCODE = 'cpcode'
ns = Netstorage(NS_HOSTNAME, NS_KEYNAME, NS_KEY, ssl=False)
def recursive(items):
for item in items:
if item.get('type') == 'file':
print("{0} {1}".format(
item.get('md5'),
items.attrib['directory'][7:] + "/" + item.get('name')))
# print("{0} {1} {2} {3} {4}".format(
# items.attrib['directory'] + "/" + item.get('name'),
# item.get('size'),
# item.get('md5'),
# item.get('mtime'),
# item.get('type')
# ))
elif item.get('type') == 'dir':
# print("{0} {1} {2} {3} {4}".format(
# items.attrib['directory'] + "/" + item.get('name'),
# item.get('bytes'),
# item.get('files'),
# item.get('mtime'),
# item.get('type')
mkdir /123456/dir1
rmdir: to delete /123456/dir1 (directory needs to be empty)
rmdir /123456/dir1
"""
usage = "Usage: python cms_netstorage.py -H [hostname] -k [keyname] -K [key] -action [action_options] .."
parser = NetstorageParser(usage=usage, epilog=action_list)
parser.add_option("-H", "--host", dest="hostname", help="Netstorage API hostname ex) xxx-nsu.akamaihd.net")
parser.add_option("-k", "--keyname", dest="keyname", help="Netstorage API keyname ex) xxxxx")
parser.add_option("-K", "--key", dest="key", help="Netstorage API key ex) xxxxxxxxxxxxx")
parser.add_option("-a", "--action", dest="action")
(options, args) = parser.parse_args()
if options.hostname and options.keyname and options.key and options.action:
ns = Netstorage(options.hostname, options.keyname, options.key)
try:
res = None
if options.action == "delete":
ok, res = ns.delete(args[0])
elif options.action == "dir":
ok, res = ns.dir(args[0])
elif options.action == "download":
ok, res = ns.download(args[0], args[1])
elif options.action == "du":
ok, res = ns.du(args[0])
elif options.action == "mkdir":
ok, res = ns.mkdir(args[0])
elif options.action == "mtime":
ok, res = ns.mtime(args[0], args[1])
dest='hostname',
help='Netstorage API hostname ex) xxx-nsu.akamaihd.net')
parser.add_option('-k',
'--keyname',
dest='keyname',
help='Netstorage API keyname ex) xxxxx')
parser.add_option('-K',
'--key',
dest='key',
help='Netstorage API key ex) xxxxxxxxxxxxx')
parser.add_option('-a', '--action', dest='action')
(options, args) = parser.parse_args()
if options.hostname and options.keyname and options.key and options.action:
ns = Netstorage(options.hostname, options.keyname, options.key)
try:
res = None
if options.action == 'delete':
ok, res = ns.delete(args[0])
elif options.action == 'dir':
ok, res = ns.dir(args[0])
elif options.action == 'download':
ok, res = ns.download(args[0], args[1])
elif options.action == 'du':
ok, res = ns.du(args[0])
elif options.action == 'mkdir':
ok, res = ns.mkdir(args[0])
elif options.action == 'mtime':
ok, res = ns.mtime(args[0], args[1])
from smart_open import open as sopen
from dotenv import load_dotenv
env_path = os.path.join(os.path.dirname(__file__), '.env')
load_dotenv(env_path)
#netstorage
root = os.getenv("NS_PATH")
if not root.startswith("/"):
root = "/{0}".format(root)
if root.endswith("/"):
root = root[:-1]
ns_host = os.getenv("NS_HOST")
key = os.getenv("NS_KEY")
keyname = os.getenv("NS_KEYNAME")
ns = Netstorage(ns_host, keyname, key)
#s3
s3_bucket = os.getenv("S3_BUCKET")
session = boto3.Session(
aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY")
)
#threads
jobs = int(os.getenv("JOBS"))
semaphore = threading.Semaphore(jobs)
threads = []
prism = {}
def setup_logger(name, file):
parser.add_option('-f',
'--folder',
action='store_true',
dest='folder',
default=False,
help='See directory(folder) only')
parser.add_option('-o',
'--output',
action='store_true',
dest='output',
default=False,
help='Save to file')
(options, args) = parser.parse_args()
nsInfo = data[options.alias]
ns = Netstorage(nsInfo['hostname'], nsInfo['keyname'], nsInfo['key'])
path = options.path
if path:
if path == '/':
path == ''
elif not path.startswith('/'):
path = '/' + path
if path.endswith('/'):
path = path[:-1]
else:
path = ''
path = '/{0}{1}'.format(nsInfo['cpcode'], path)
ok, res = ns.dir(path)