def _sort_gen(self):
gen_list = []
for i, generator in enumerate(self.generator):
gen_list.append((generator, i))
sorted_gen_index = sorted(gen_list, key=lambda ele: ele[0])
self.sorted_index = [ele[1] for ele in sorted_gen_index]
get_logger().info(self.sorted_index)
def _sort_gen(self):
gen_list = []
for i, generator in enumerate(self.generator):
gen_list.append((generator, i))
sorted_gen_index = sorted(gen_list, key=lambda ele: ele[0])
self.sorted_index = [ele[1] for ele in sorted_gen_index]
get_logger().info(self.sorted_index)
def gen_q(data_ndarray, ndim):
"""
generate q using GF^8
:param data_ndarray: the data ndarray
:param ndim: real dim
:return: q_ndarray with shape=(1, byte_ndarray.shape[1])
"""
transposed = np.transpose(data_ndarray)
# print(data_ndarray.shape)
get_logger().info('transposed\n{}'.format(transposed))
gf = GF()
q_list = []
for _1darray in transposed:
bv_list = []
for i, arr_val in enumerate(_1darray):
res_i = gf.multiply(gf.generator[i % gf.circle], int(arr_val))
# print('i={}, arr_val={}, res_i={}'.format(i, arr_val, res_i))
bv_list.append(res_i)
# map(lambda i: print(i), bv_list)
q_value = reduce(operator.xor, bv_list)
q_list.append(q_value)
arr = np.array(q_list, ndmin=ndim, dtype=config.BYTE_TYPE)
get_logger().info("arr={}".format(arr))
# assert arr.shape[1] == data_ndarray.shape[1]
return arr
def gen_q(data_ndarray, ndim):
"""
generate q using GF^8
:param data_ndarray: the data ndarray
:param ndim: real dim
:return: q_ndarray with shape=(1, byte_ndarray.shape[1])
"""
transposed = np.transpose(data_ndarray)
# print(data_ndarray.shape)
get_logger().info('transposed\n{}'.format(transposed))
gf = GF()
q_list = []
for _1darray in transposed:
bv_list = []
for i, arr_val in enumerate(_1darray):
res_i = gf.multiply(gf.generator[i % gf.circle], int(arr_val))
# print('i={}, arr_val={}, res_i={}'.format(i, arr_val, res_i))
bv_list.append(res_i)
# map(lambda i: print(i), bv_list)
q_value = reduce(operator.xor, bv_list)
q_list.append(q_value)
arr = np.array(q_list, ndmin=ndim, dtype=config.BYTE_TYPE)
get_logger().info("arr={}".format(arr))
# assert arr.shape[1] == data_ndarray.shape[1]
return arr
def __gen_raid_array(self, byte_ndarray):
"""generate full ndarray from raw data ndarray
"""
# calculate parity and append
parity = utils.gen_p(byte_ndarray, ndim=2)
write_array = np.concatenate([byte_ndarray, parity])
get_logger().info('write_array=\n{}'.format(write_array))
return write_array
def dump_bitvector(bv, display_base='x', display_width=2):
"""
helper method to dump bitvector
:param bv:
:param display_base:
:param display_width:
:return:
"""
int_val = int(str(bv), base=2)
get_logger().info('{:0{width}{base}}'.format(int_val, base=display_base, width=display_width))
def test_from_content(r6):
get_logger().warning("testing from content")
original_content = b'good_morning\x03_sir_yes_great\x01\x02'
# original_content = b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13'
data_fname = 'my.dat'
# r6.write(original_content, data_fname)
# r6.recover_d_or_p(data_fname, error_index)
# r6.recover_d_p(data_fname, 1)
# r6.recover_2d(data_fname, 0, 1)
# r6_content = r6.read(data_fname, len(original_content))
# assert r6_content == original_content
r6.detect_corruption(data_fname)
def test_from_content(r6):
get_logger().warning("testing from content")
original_content = b'good_morning\x03_sir_yes_great\x01\x02'
# original_content = b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13'
data_fname = 'my.dat'
# r6.write(original_content, data_fname)
# r6.recover_d_or_p(data_fname, error_index)
# r6.recover_d_p(data_fname, 1)
# r6.recover_2d(data_fname, 0, 1)
# r6_content = r6.read(data_fname, len(original_content))
# assert r6_content == original_content
r6.detect_corruption(data_fname)
def log_generator(self, result):
"""
log_g operation
:param result: the value to be log
:return: int result
"""
get_logger().info(type(result))
assert type(result) in (int, BitVector)
if isinstance(result, BitVector):
result = result.int_val()
for i in range(self.circle):
if self.generator[i].int_val() == result:
return i
def swap(self, byte_ndarray):
"""
swap the data from the final array to the destination slot
:param byte_ndarray:
:return:
"""
# get_logger().info('before\n{}'.format(byte_ndarray))
for i in xrange(self.N - 1):
j = self.get_parity_index(i)
get_logger().info('{}, {}'.format(byte_ndarray[i][j], byte_ndarray[self.N - 1][j]))
byte_ndarray[i][j], byte_ndarray[self.N - 1][j] = byte_ndarray[self.N - 1][j], byte_ndarray[i][j]
# get_logger().info('after\n{}'.format(byte_ndarray))
return byte_ndarray
def swap(self, byte_ndarray):
"""
swap the data from the final array to the destination slot
:param byte_ndarray:
:return:
"""
# get_logger().info('before\n{}'.format(byte_ndarray))
for i in xrange(self.N - 1):
j = self.get_parity_index(i)
get_logger().info('{}, {}'.format(byte_ndarray[i][j],
byte_ndarray[self.N - 1][j]))
byte_ndarray[i][j], byte_ndarray[self.N - 1][j] = byte_ndarray[
self.N - 1][j], byte_ndarray[i][j]
# get_logger().info('after\n{}'.format(byte_ndarray))
return byte_ndarray
def init_generator(self):
"""
pre-compute the lookup table of the generator
"""
get_logger().info('init generator')
self.generator = []
res = BitVector(intVal=1)
base = BitVector(intVal=2)
counter = 0
while True:
self.generator.append(res)
self.dump_bitvector(res)
counter += 1
if counter >= self.circle:
break
res = self.multiply(res, base)
assert len(set(self.generator)) == self.circle
def init_generator(self):
"""
generate look up generator table
"""
self.generator = []
counter = 0
res = 1
base = 2
while True:
self.generator.append(res)
counter += 1
if counter >= self.circle:
break
res = self.multiply(res, base)
assert len(self.generator) == self.circle
self._sort_gen()
for g in self.generator:
get_logger().info('{:0{width}{base}}'.format(g, base='x', width=2))
def init_generator(self):
"""
generate look up generator table
"""
self.generator = []
counter = 0
res = 1
base = 2
while True:
self.generator.append(res)
counter += 1
if counter >= self.circle:
break
res = self.multiply(res, base)
assert len(self.generator) == self.circle
self._sort_gen()
for g in self.generator:
get_logger().info('{:0{width}{base}}'.format(g, base='x', width=2))
def test_from_data_file(r6):
import driver
def _corrupt(fname, index, size):
get_logger().warning("corrupting disk {}".format(index))
error_fpath = r6.get_real_name(index, fname)
error_content = os.urandom(size)
utils.write_content(error_fpath, error_content)
def _corrupt2(fname, indexes, size):
for index in indexes:
_corrupt(fname, index, size)
data_fname = 'data3'
SIZE = 32768
driver.gen_rnd_file(data_fname, SIZE, 'text')
fpath = os.path.join(config.root, 'data3')
original_content = utils.read_content(fpath)
r6.write(original_content, data_fname)
r6.detect_corruption(data_fname)
for error_index in [0, 3, r6.N - 2, r6.N - 1]:
error_size = SIZE / 13
_corrupt(data_fname, error_index, error_size)
found_error_index = r6.detect_corruption(data_fname)
if found_error_index is not None:
get_logger().warning("recover disk {}".format(error_index))
assert found_error_index == error_index
if found_error_index < r6.N - 1:
r6.recover_d_or_p(data_fname, found_error_index)
else:
r6.recover_q(data_fname)
r6.detect_corruption(data_fname)
#####################################################
get_logger().warning("testing recover_d_q")
error_indexes = [4, r6.N - 1]
size = SIZE / (r6.N - 4)
_corrupt2(data_fname, error_indexes, size)
r6.recover_d_q(data_fname, error_indexes[0])
r6.detect_corruption(data_fname)
#####################################################
get_logger().warning("testing recover_2d")
error_indexes = [0, 1]
size = SIZE / (r6.N + 2)
_corrupt2(data_fname, error_indexes, size)
r6.recover_2d(data_fname, error_indexes[0], error_indexes[1])
r6.detect_corruption(data_fname)
#####################################################
get_logger().warning("testing recover_d_p")
error_indexes = [0, r6.N - 2]
size = SIZE / (r6.N - 2)
_corrupt2(data_fname, error_indexes, size)
r6.recover_d_p(data_fname, error_indexes[0])
r6.detect_corruption(data_fname)
def test_from_data_file(r6):
import driver
def _corrupt(fname, index, size):
get_logger().warning("corrupting disk {}".format(index))
error_fpath = r6.get_real_name(index, fname)
error_content = os.urandom(size)
utils.write_content(error_fpath, error_content)
def _corrupt2(fname, indexes, size):
for index in indexes:
_corrupt(fname, index, size)
data_fname = 'data3'
SIZE = 32768
driver.gen_rnd_file(data_fname, SIZE, 'text')
fpath = os.path.join(config.root, 'data3')
original_content = utils.read_content(fpath)
r6.write(original_content, data_fname)
r6.detect_corruption(data_fname)
for error_index in [0, 3, r6.N - 2, r6.N - 1]:
error_size = SIZE / 13
_corrupt(data_fname, error_index, error_size)
found_error_index = r6.detect_corruption(data_fname)
if found_error_index is not None:
get_logger().warning("recover disk {}".format(error_index))
assert found_error_index == error_index
if found_error_index < r6.N - 1:
r6.recover_d_or_p(data_fname, found_error_index)
else:
r6.recover_q(data_fname)
r6.detect_corruption(data_fname)
#####################################################
get_logger().warning("testing recover_d_q")
error_indexes = [4, r6.N - 1]
size = SIZE / (r6.N - 4)
_corrupt2(data_fname, error_indexes, size)
r6.recover_d_q(data_fname, error_indexes[0])
r6.detect_corruption(data_fname)
#####################################################
get_logger().warning("testing recover_2d")
error_indexes = [0, 1]
size = SIZE / (r6.N + 2)
_corrupt2(data_fname, error_indexes, size)
r6.recover_2d(data_fname, error_indexes[0], error_indexes[1])
r6.detect_corruption(data_fname)
#####################################################
get_logger().warning("testing recover_d_p")
error_indexes = [0, r6.N - 2]
size = SIZE / (r6.N - 2)
_corrupt2(data_fname, error_indexes, size)
r6.recover_d_p(data_fname, error_indexes[0])
r6.detect_corruption(data_fname)
def gen_rnd_file(fname, size, content_type):
"""
randomly generate data chunk
:param fname: the name denoting the data
:param size: the size of the data chunk
:param content_type: can be 'text' or others; if it's 'text', make the chunk only contain ascii letters
:return:
"""
if content_type == 'text':
# noinspection PyUnusedLocal
content = ''.join([random.choice(string.ascii_letters) for i in xrange(size)])
else:
content = os.urandom(size)
fpath = os.path.join(config.root, fname)
if os.path.isfile(fpath):
file_size = os.stat(fpath).st_size
if file_size == size:
get_logger().warning('fname={} with size={} exists'.format(fname, size))
return
get_logger().warning('generating fname={} with size={}'.format(fname, size))
utils.write_content(fpath, content)
def _gen_ndarray_from_content(self, content, num):
"""
convert the content into numpy 2-darray
:param content: read content
:param num: 1st dimension size
:return:
"""
# gen N-1 length empty list
content_list = [[] for i in repeat(None, num)]
for i in xrange(len(content)):
mod_i = i % num
content_list[mod_i].append(content[i])
byte_list = []
length = len(sorted(content_list, key=len, reverse=True)[0])
# fill 0
for content in content_list:
current_str_list = [ord(s) for s in content] + [self.ZERO] * (length - len(content))
byte_list.append(current_str_list)
byte_nparray = np.array(byte_list, dtype=self.BYTE_TYPE)
get_logger().info('byte_array'.format(byte_nparray))
return byte_nparray
def _gen_ndarray_from_content(self, content, num):
"""
convert the content into numpy 2-darray
:param content: read content
:param num: 1st dimension size
:return:
"""
# gen N-1 length empty list
content_list = [[] for i in repeat(None, num)]
for i in xrange(len(content)):
mod_i = i % num
content_list[mod_i].append(content[i])
byte_list = []
length = len(sorted(content_list, key=len, reverse=True)[0])
# fill 0
for content in content_list:
current_str_list = [ord(s) for s in content
] + [self.ZERO] * (length - len(content))
byte_list.append(current_str_list)
byte_nparray = np.array(byte_list, dtype=self.BYTE_TYPE)
get_logger().info('byte_array'.format(byte_nparray))
return byte_nparray
def gen_rnd_file(fname, size, content_type):
"""
randomly generate data chunk
:param fname: the name denoting the data
:param size: the size of the data chunk
:param content_type: can be 'text' or others; if it's 'text', make the chunk only contain ascii letters
:return:
"""
if content_type == 'text':
# noinspection PyUnusedLocal
content = ''.join(
[random.choice(string.ascii_letters) for i in xrange(size)])
else:
content = os.urandom(size)
fpath = os.path.join(config.root, fname)
if os.path.isfile(fpath):
file_size = os.stat(fpath).st_size
if file_size == size:
get_logger().warning('fname={} with size={} exists'.format(
fname, size))
return
get_logger().warning('generating fname={} with size={}'.format(
fname, size))
utils.write_content(fpath, content)
def duration(then):
t = time.time() - then
get_logger().info('takes {}'.format(t))
return t
'--name',
help='container name regex pattern',
dest='name',
required=True)
parser.add_argument(
'-s',
'--sleep',
type=int,
default=30,
help='sleep duration in seconds between consecutive actions',
dest='sleep')
parser.add_argument("-v",
"--verbose",
help="increase output verbosity",
action="store_true")
logger = log_helper.get_logger()
def signal_handler(signal, frame):
"""
handle the Ctrl-C keyboard interrupt by cleanly exiting
"""
logger.debug("Exiting")
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
def filter_containers_by_name(pattern):
"""
#!/usr/bin/env python
import yaml
from log_helper import get_logger
l = get_logger('config')
#this class is the parser for yml file
def parser():
filename = "../settings.yml"
with open(filename, 'r') as stream:
try:
l.info('Parsing Config Files')
l.info('Opening {}'.format(filename))
l.info('{} Successfully Read'.format(filename))
val = yaml.load(stream)
# print(val)
except yaml.YAMLError as exc:
l.fatal('Error parsing file {filename}: {exc}. Please fix and try '
'again.'.format(filename=filename, exc=exc))
# print(exc)
return val
def ethosUrl():
def logger(msg):
global _log_path
log_helper.get_logger(_log_path).info(msg)
class SalesForce():
log = get_logger('SalesForce')
__auth_params__ = set(
['grant_type', 'client_id', 'client_secret', 'username', 'password'])
_service_path = '/services/data/v40.0'
def __init__(self, **kwargs):
auth_params = {}
if self.__auth_params__.issubset(set(kwargs.keys())):
for param in self.__auth_params__:
auth_params.update({param: kwargs.get(param, None)})
self.session = requests.Session()
self.session.headers.update({
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
})
r = self.session.post(
"https://login.salesforce.com/services/oauth2/token",
params=auth_params)
self._access_token = r.json().get("access_token", None)
self.session.headers[
'Authorization'] = f'Bearer {self._access_token}'
self.session.cookies.set(name='sid',
value=kwargs['cookie_sid'],
domain=kwargs['cookie_domain'])
self.session.cookies.set(
name='atlassian.xsrf.token',
value=
'B2YH-NCM1-ID0V-D60M_8284e4e214fc14e59ad9ea5bbff670c9b3d20255_lin',
domain='jira.devfactory.com')
self.session.cookies.set(
name='JSESSIONID',
value='E70BC00650914C5DE7986E27DFCE0F47.jira-prod-node-4',
domain='jira.devfactory.com')
self.session.cookies.set(name='route',
value='1574861615.812.134449.885484',
domain='jira.devfactory.com')
self.instance_url = r.json().get("instance_url", None)
def get_image(self, url):
response = self.session.get(url, timeout=(2, 25))
return response.content
def _api_call(self, action, parameters={}, method='get', data={}):
"""
Helper function to make calls to Salesforce REST API.
Parameters: action (the URL), URL params, method (get, post or patch), data for POST/PATCH.
"""
headers = {
'Content-type': 'application/json',
'Accept-Encoding': 'gzip',
'Authorization': 'Bearer %s' % self._access_token
}
if self._service_path not in action:
action = self._service_path + action
if method == 'get':
r = self.session.request(method,
self.instance_url + action,
headers=headers,
params=parameters)
elif method in ['post', 'patch']:
r = self.session.request(method,
self.instance_url + action,
headers=headers,
json=data,
params=parameters)
else:
# other methods not implemented in this example
raise ValueError('Method should be get or post or patch.')
self.log.debug('Debug: API %s call: %s' % (method, r.url))
if r.status_code < 300:
if method == 'patch':
return None
else:
if 'application/json' in r.headers.get('content-type'):
return r.json()
else:
return r.content
else:
raise Exception('API error when calling %s : %s' %
(r.url, r.content))
def discover_fields(self, sf_object):
response = self._api_call(f'/sobjects/{sf_object}/describe')
if not response.get('queryable', False):
raise Exception("This object is not queryable")
return response.get('fields', None)
def get_case_comments(self, case_id):
fields = ['CreatedById', 'CreatedDate', 'CommentBody', 'IsPublished']
conditions = [f'ParentId={case_id}']
response = self.query_all('CaseComments', fields, conditions)
return response
def get_case_attachments(self, case_id):
response = self._api_call(f'/Case/{case_id}/Attachments')
return response
def download_attachment(self, file_id):
response = self._api_call(f'/sobjects/Attachment/{file_id}/body')
return response
def query_all(self, sf_object, fields, conditions=[], limit=None):
query_str = f"SELECT {', '.join(fields)} FROM {sf_object} " + (
"WHERE " + " AND ".join(conditions) if len(conditions) > 0 else
"") + (f"LIMIT {limit}" if limit else "")
response = self._api_call('/queryAll/', {'q': query_str})
next = response.get('nextRecordsUrl', None)
yield response.get('records', [])
while next:
response = self._api_call(next)
next = response.get('nextRecordsUrl', None)
yield response.get('records', [])
import json
import csv
import os
from config import ENV
from salesforce import SalesForce
from zendesk import Zendesk
from migration import MigrationItem
from log_helper import get_logger
import helpers
log = get_logger('Main')
if __name__ == "__main__":
migration_plan = json.load(open('migration_plan.json', 'r'))
sf_config = migration_plan['salesforce']
sf = SalesForce(**sf_config)
zd_config = migration_plan['zendesk']
zd = Zendesk(**zd_config)
for item in migration_plan['migration_items']:
if item['skip'] == False:
if item['type'] == 'migration_object':
migration_item = MigrationItem(sf, zd, mode=ENV, **item)
if migration_item.skip == False:
try:
migration_item.migrate()
except Exception as err:
log.critical(err)
raise err
elif item['type'] == 'script':
#!/usr/bin/env python
from pyEthOS import pyEthOS as ethos
import requests
import json
from config import ethosUrl
from log_helper import get_logger
l = get_logger('ethosapi')
def ethosapi():
PANEL_NAME = ethosUrl()
DEBUG = False
#api = ethos.EthOS_API(PANEL_NAME, debug=DEBUG)
try:
api = ethos.EthOS_API(PANEL_NAME, debug=DEBUG)
except requests.exceptions.RequestException as e:
print(e)
sys.exit(1)
#req = requests.get('http://altaiu.ethosdistro.com')
#req2 = req.json()
apiReq = api.get_summary()
def logger(mes):
log_helper.get_logger(__log__path).info(mes)
def _corrupt(fname, index, size):
get_logger().warning("corrupting disk {}".format(index))
error_fpath = r6.get_real_name(index, fname)
error_content = os.urandom(size)
utils.write_content(error_fpath, error_content)
class RAID6(RAID):
_logger = get_logger()
def __init__(self, N):
assert 4 <= N
super(RAID6, self).__init__(N)
# gf object
self.gf = GF()
def check(self, byte_ndarray):
"""
check involves p and q
:param byte_ndarray: all ndarray including p and q
:return:
"""
# check p
data_p_ndarray = byte_ndarray[:-1]
utils.check_data_p(data_p_ndarray)
# check_q
data_ndarray = byte_ndarray[:-2]
q_ndarray = byte_ndarray[-1:]
utils.check_q(data_ndarray, q_ndarray)
def read(self, fname, size):
"""
read size chunk from fname(RAID6 system)
"""
byte_ndarray = self._read_n(fname, self.N)
self.check(byte_ndarray)
data_ndarray = byte_ndarray[:-2]
flat_list = data_ndarray.ravel(1)[:size]
flat_str_list = [chr(e) for e in flat_list]
return ''.join(flat_str_list)
def recover(self, fname, exclude):
"""
since there are different cases, we raise an error indicating it should not be called
"""
raise NotImplementedError("not implemented; split into several cases")
def _get_corrupted_data_disk(self, P_star, Q_star):
"""
from P_star and Q_star, find the corrupted data disk index
procondition: P_star!={00}, Q_star!={00} (in vector sense)
"""
p0 = int(P_star[0][0])
q0 = int(Q_star[0][0])
log_p0 = self.gf.log_generator(p0)
log_q0 = self.gf.log_generator(q0)
return (log_q0 - log_p0) % self.gf.circle
def detect_corruption(self, fname):
"""
single disk corruption detection
:param fname: data name in RAID6 system
:return: corrupted disk index; for p, self.N-2; for q, self.N-1
"""
# all disks, including P, Q
byte_ndarray = self._read_n(fname, self.N)
data_ndarray = byte_ndarray[:-2]
self._logger.info("byte_ndarray=\n{}".format(byte_ndarray))
P = byte_ndarray[-2:-1]
self._logger.info("p={}".format(P))
Q = byte_ndarray[-1]
self._logger.info("Q={}".format(Q))
P_prime = utils.gen_p(data_ndarray, ndim=2)
self._logger.info("P_prime={}".format(P_prime))
Q_prime = utils.gen_q(data_ndarray, ndim=2)
self._logger.info("Q_prime={}".format(Q_prime))
P_star = np.bitwise_xor(P, P_prime)
Q_star = np.bitwise_xor(Q, Q_prime)
P_nonzero = np.count_nonzero(P_star)
Q_nonzero = np.count_nonzero(Q_star)
if P_nonzero == 0 and Q_nonzero == 0:
print("no corruption")
return None
elif P_nonzero == 0 and Q_nonzero != 0:
print("Q corruption")
return self.N - 1
elif P_nonzero != 0 and Q_nonzero == 0:
print("P corruption")
return self.N - 2
else:
index = self._get_corrupted_data_disk(P_star, Q_star)
print("data disk {} corruption".format(index))
return index
def recover_d_or_p(self, fname, index):
"""
recover data drive or 'p' drive, simply using XOR
:param fname: data name
:param index: data disk or p disk index
:return:
"""
assert 0 <= index < self.N - 1
byte_ndarray = self._read_n(fname, self.N - 1, exclude=index)
parity = utils.gen_p(byte_ndarray, ndim=1)
content = self._1darray_to_str(parity)
fpath = self.get_real_name(index, fname)
utils.write_content(fpath, content)
# check data or p
read_ndarray = self._read_n(fname, self.N - 1)
utils.check_data_p(read_ndarray)
def recover_q(self, fname):
"""
recover 'q' drive, recompute
:param fname: data name
:return:
"""
byte_ndarray = self._read_n(fname, self.N - 2)
q_ndarray = utils.gen_q(byte_ndarray, ndim=2)
assert q_ndarray.ndim == 2
new_num = q_ndarray.shape[1]
q_ndarray.shape = (new_num, )
content = self._1darray_to_str(q_ndarray)
fpath = self.get_real_name(self.N - 1, fname)
utils.write_content(fpath, content)
def recover_d_q(self, fname, index):
"""
recover data/'p' drive (index) and 'q' drive: firstly using XOR to recover data drive, then recompute q
:param fname: data name
:param index: corrupted data disk index
:return:
"""
self.recover_d_or_p(fname, index)
self.recover_q(fname)
def recover_2d(self, fname, x, y):
"""
recover data drives (x and y)
:param fname: data name
:param x: corrupted data disk index
:param y: corrupted data disk index
:return:
"""
assert 0 <= x < self.N - 2
assert 0 <= y < self.N - 2
assert x != y
byte_ndarray = self._read_n(fname, self.N, exclude=[x, y])
DD = byte_ndarray[:-2]
P = byte_ndarray[-2:-1]
Q = byte_ndarray[-1:]
# Pxy
Pxy = utils.gen_p(DD, ndim=2)
# Qxy
Qxy = utils.gen_q(DD, ndim=2)
# Axy, Bxy
A = self.gf.Axy(x, y)
B = self.gf.Bxy(x, y)
# Dx
first = utils.gf_a_multiply_list(A, utils.gf_1darray_add(P, Pxy))
second = utils.gf_a_multiply_list(B, utils.gf_1darray_add(Q, Qxy))
Dx = utils.gf_1darray_add(np.array(first, dtype=config.BYTE_TYPE),
np.array(second, dtype=config.BYTE_TYPE))
Dx_content = self._1darray_to_str(Dx)
x_fpath = self.get_real_name(x, fname)
utils.write_content(x_fpath, Dx_content)
# Dy
Dy = utils.gf_1darray_add(P ^ Pxy, Dx)
Dy_content = self._1darray_to_str(Dy)
y_fpath = self.get_real_name(y, fname)
utils.write_content(y_fpath, Dy_content)
def recover_d_p(self, fname, index):
"""
recover data drive (index) and 'p' drive
:param fname: data name
:param index: data disk index
:return:
"""
assert 0 <= index < self.N - 2
byte_ndarray = self._read_n(fname, self.N, exclude=index)
DD = byte_ndarray[:-2]
Q = byte_ndarray[-1:]
# Dx
Qx = utils.gen_q(DD, ndim=2)
g_x_inv = self.gf.generator[(self.gf.circle - index) % self.gf.circle]
###
_add_list = utils.gf_1darray_add(Q, Qx)
Dx_list = utils.gf_a_multiply_list(g_x_inv, _add_list)
###
Dx_content = ''.join(chr(i) for i in Dx_list)
x_fpath = self.get_real_name(index, fname)
utils.write_content(x_fpath, Dx_content)
# p
Dx = np.array(Dx_list, ndmin=2)
assert Dx.shape[1] == byte_ndarray.shape[1]
# update firstly
DD[index] = Dx
P = utils.gen_p(DD, ndim=1)
assert P.shape[0] == byte_ndarray.shape[1]
# do not need to update DD
P_content = self._1darray_to_str(P)
P_path = self.get_real_name(self.N - 2, fname)
utils.write_content(P_path, P_content)
def write(self, content, fname):
"""
write content to fname(in RAID6 system)
"""
byte_ndarray = self._gen_ndarray_from_content(content, self.N - 2)
p_ndarray = utils.gen_p(byte_ndarray, ndim=2)
q_ndarray = utils.gen_q(byte_ndarray, ndim=2)
write_ndarray = np.concatenate([byte_ndarray, p_ndarray, q_ndarray])
self._write_n(fname, write_ndarray, self.N)
class MigrationItem():
log = get_logger('MigrationPlan')
__data_folder__ = 'migration_data'
def __init__(self, sf, zd, **kwargs):
self.env_mode = kwargs.get('mode', 'dev')
self._sf = sf
self._zd = zd
self.zd_custom_endpoint = kwargs.get('zd_custom_endpoint', None)
self.skip = kwargs.get('skip', True)
self.sf_object = kwargs.get('sf_object', None)
self.payload_file = kwargs.get('file', None)
self.zd_object = kwargs['zd_object']
self.limit = kwargs.get('limit', None)
self.upsert = kwargs.get('upsert', True)
self.create_mapping = kwargs.get('create_mapping', True)
self.after_download = kwargs.get('after_download', [])
self.bulk_create_or_update = kwargs.get('bulk_create_or_update', False)
self.sf_fields = [
field for field in kwargs.get('sf_fields', [])
if type(field) == str
]
self.sf_join_fields = [
field for field in kwargs.get('sf_fields', [])
if type(field) == dict
]
if len(self.sf_fields) == 0 and self.sf_object:
self.sf_fields = [
field['name'] for field in sf.discover_fields(self.sf_object)
]
self.sf_conditions = kwargs.get('sf_conditions', [])
self.fields_mapping = kwargs.get('fields_mapping', [])
self.force_download = kwargs.get('force_download', True)
self.batch_folder = f'{self.__data_folder__}/{self.sf_object or self.zd_object}/batches'
self.export_file = f'{self.__data_folder__}/{self.sf_object}/data.csv'
self.data_file = f'{self.__data_folder__}/{self.sf_object}/data.json'
self.id_mapping_file = f'{self.__data_folder__}/{self.sf_object}/{self.env_mode}-mapping.json'
self.errors_file = f'{self.__data_folder__}/{self.sf_object or self.zd_object}/errors.json'
self.mapping_errors_file = f'{self.__data_folder__}/{self.sf_object or self.zd_object}/mapping-errors.json'
def log_mapping_error(self, obj, source, key, value):
json_payload = [{
'object': obj,
'source': source,
'key': key,
'value': value
}]
if not os.path.exists(self.mapping_errors_file):
json.dump(json_payload, open(self.mapping_errors_file, 'w+'))
else:
log_json = json.load(open(self.mapping_errors_file, 'r'))
log_json.extend(json_payload)
json.dump(log_json, open(self.mapping_errors_file, 'w+'))
def __build_directory(self):
os.makedirs(
f'{self.__data_folder__}/{self.sf_object or self.zd_object}/batches',
exist_ok=True)
def __parse_sf_row(self, row, fields=None):
parsed_row = {}
for field in (fields or self.sf_fields):
if len(field.split('.')) == 1:
value = row[field]
elif row[field.split('.')[0]] is not None:
field_key = field.split('.')[0]
child_key = field.split('.')[-1]
value = row[field_key].get(child_key, None)
else:
value = None
parsed_row.update({field: value})
return parsed_row
def get_data(self):
if self.sf_object is None and self.payload_file:
data = json.load(open(self.payload_file, 'r',
encoding='utf-8-sig'))
else:
if os.path.exists(self.data_file) is False:
self.download_data()
data = json.load(open(self.data_file, 'r', encoding='utf-8-sig'))
return data
def download_data(self):
self.__build_directory()
self.log.info(
f'Downloading data from SalesForce for {self.sf_object} object')
csv_writer = csv.DictWriter(open(self.export_file,
'w+',
newline='',
encoding="utf-8-sig"),
self.sf_fields,
extrasaction='ignore')
csv_writer.writeheader()
data = []
for rows in self._sf.query_all(
self.sf_object,
[field for field in self.sf_fields if type(field) == str],
self.sf_conditions,
limit=self.limit):
if len(rows) > 0:
for row in rows:
parsed_row = self.__parse_sf_row(row)
for join_field in self.sf_join_fields:
self.log.info(
f'Getting {join_field["sf_object"]} for {self.sf_object}:{row["Id"]} object'
)
parsed_row.update({join_field['sf_object']: []})
for join_rows in self._sf.query_all(
join_field['sf_object'],
join_field['sf_fields'],
[
eval(condition, {'row': row})
for condition in join_field['sf_conditions']
]):
for join_row in join_rows:
parsed_row[join_field['sf_object']].append(
self.__parse_sf_row(
join_row, join_field['sf_fields']))
data.append(parsed_row)
csv_writer.writerow(parsed_row)
json.dump(data, open(self.data_file, 'w+', encoding='utf-8-sig'))
self.log.info('Done.')
def on_after_download(self):
for func in self.after_download:
self.log.info(f'Evaluating function after download {func}')
eval(func)
def _update_zd_obj(self, json):
result = self._zd.put(
f'/{self.zd_custom_endpoint or self.zd_object}/update_many.json',
json=json,
response_container='job_status')
status = result['status']
while status in ['queued', 'working']:
job_url = result['url']
result = self._zd.get(job_url, 'job_status')
status = result['status']
if status in ['queued', 'working']:
time.sleep(10)
return result.get('results', None)
def _create_zd_obj(self, json):
result = None
while result is None:
result = self._zd.post(
f'/{self.zd_custom_endpoint or self.zd_object}/create_many.json',
data=json,
response_container='job_status')
if 'status_code' in result and result.status_code == 429:
wait_time = int(result.headers['Retry-After'])
result = None
time.sleep(wait_time)
continue
status = result['status']
while status in ['queued', 'working']:
job_url = result['url']
result = self._zd.get(job_url, 'job_status')
status = result['status']
if status in ['queued', 'working']:
message = result.get('message', None)
if message:
self.log.info(message)
else:
self.log.info('Waiting reponse...')
time.sleep(10)
return result.get('results', None)
def _create_or_update_zd_obj(self, json):
result = self._zd.post(
f'/{self.zd_custom_endpoint or self.zd_object}/create_or_update_many.json',
data=json,
response_container='job_status')
status = result['status'] if 'status' in result else 'error'
while status in ['queued', 'working', 'error']:
job_url = result['url']
result = self._zd.get(job_url, 'job_status')
status = result['status']
if status in ['queued', 'working']:
self.log.info('Waiting reponse...')
time.sleep(10)
return result.get('results', None)
def get_zd_payload(self):
self.log.info(f'Building Zendesk {self.zd_object} object payload')
data = self.get_data()
total = len(data)
if self.sf_object:
if os.path.exists(
os.path.join(self.__data_folder__, self.sf_object,
'payload.json')):
return json.load(
open(
os.path.join(self.__data_folder__, self.sf_object,
'payload.json'), 'r'))
payload = {self.zd_object: []}
for idx, fields in enumerate(data):
self.log.info(f'Processing... {idx}/{total}')
mapped = {}
for map_item in self.fields_mapping:
field_type = map_item['field']['type']
field_key = map_item['field']['key']
field_value = map_item['value']
value_type = map_item.get('type', None)
if value_type == 'sf_field':
field_value = eval(field_value, {
"helpers": helpers,
"self": self
}, {
key.replace('.', '_'): value
for key, value in fields.items()
})
elif value_type == 'mapping':
mapping_source = map_item.get('source', None)
mapping_key = field_value
if mapping_source and mapping_key:
mapping = self._get_mapping(mapping_source)
if mapping:
field_value = mapping.get(
fields[mapping_key], None)
if field_value is None:
self.log_mapping_error(
fields, mapping_source, mapping_key,
fields[mapping_key])
self.log.warning(
f'No Mapping found for {field_key} in {mapping_source} |\tkey:{fields[mapping_key]}'
)
else:
fallback_value = map_item.get(
'fallback_value', None)
field_value = fallback_value
self.log_mapping_error(fields, mapping_source,
mapping_key,
fields[mapping_key])
self.log.warning(
f'No Mapping found for {field_key} in {mapping_source}. Fallbak value is used instead'
)
else:
field_value = map_item['value']
if field_value is not None:
if field_type == 'standard':
mapped.update({field_key: field_value})
else:
if mapped.get(field_type, None):
mapped[field_type].update(
{field_key: field_value})
else:
mapped.update(
{field_type: {
field_key: field_value
}})
payload[self.zd_object].append(mapped)
self.log.info(f'Done.')
json.dump(
payload,
open(
os.path.join(self.__data_folder__, self.sf_object,
'payload.json'), 'w+'))
return payload
return data
def _create_batch_payload(self):
batches = []
payload = self.get_zd_payload()
slices = round(len(payload[self.zd_object]) / 100 + 0.5)
slice_size = 100
for i in range(slices):
batches.append({
self.zd_object:
payload[self.zd_object][i * slice_size:(i + 1) * slice_size]
})
return batches
def _create_batch_list(self, items):
batches = []
slices = round(len(items) / 100 + 0.5)
slice_size = 100
for i in range(slices):
batches.append(
list(items)[i * slice_size:slice_size + i * slice_size])
return batches
def _get_mapping(self, source=None):
if source:
file = f'{self.__data_folder__}/{source}/{self.env_mode}-mapping.json'
return self.__get_json_file(file)
if os.path.exists(self.id_mapping_file):
return self.__get_json_file(self.id_mapping_file)
return {}
def __get_json_file(self, file):
if os.path.exists(file):
f = open(file, 'r')
data = f.read()
f.close()
return json.loads(data)
return None
def get_sync_diff(self):
mapping = self._get_mapping()
if mapping:
data = self.get_data()
new_obj_ids = set([item['Id'] for item in data])
old_obj_ids = set([key for key in mapping.keys()])
diff = old_obj_ids.difference(new_obj_ids)
return diff
return None
def migrate(self):
self.__build_directory()
self.log.info(f'Starting Migration of {self.zd_object} Object')
import_payload_file = f'migration_data/{self.sf_object}/import_payload.json'
if self.sf_object:
if self.force_download == True:
self.download_data()
self.on_after_download()
if self.payload_file:
payload_batches = self._create_batch_payload()
else:
if os.path.exists(import_payload_file):
payload_batches = json.load(
open(import_payload_file, 'r', encoding='utf-8-sig'))
else:
payload_batches = self._create_batch_payload()
json.dump(
payload_batches,
open(import_payload_file, 'w+', encoding='utf-8-sig'))
obj_mapping = {}
obj_error = {self.zd_object: []}
self.log.info(f'Uploading data to zendesk')
for idx, batch in enumerate(payload_batches):
batch_mapping = {}
batch_error = {self.zd_object: []}
batch_mapping_file = os.path.join(self.batch_folder,
f'batch_success_{idx}.json')
batch_error_file = os.path.join(self.batch_folder,
f'batch_error_{idx}.json')
if os.path.exists(batch_mapping_file):
self.log.info(f'Skipped batch {idx}. Already processed.')
obj_mapping.update(
json.load(
open(batch_mapping_file, 'r', encoding='utf-8-sig')))
if os.path.exists(batch_error_file):
obj_error[self.zd_object].extend(
json.load(
open(batch_error_file, 'r', encoding='utf-8-sig')))
continue
batch_success = len(batch[self.zd_object])
self.log.info(f'\tBatch {idx} of {len(payload_batches)}')
json.dump({}, open(batch_mapping_file, 'w+'))
if self.bulk_create_or_update == True:
value = batch
key = 'bulk_create_update'
result = self._create_or_update_zd_obj(batch)
if result:
for i in range(len(value[self.zd_object])):
id = result[i].get('id', None)
if id:
if self.create_mapping == True:
batch_mapping.update({
value[self.zd_object][i]['external_id']:
id
})
self.log.info(
f"[{self.zd_object}][{key}] SalesForce Id: {value[self.zd_object][i]['external_id']}\tZendesk Id:{id}"
)
else:
if key != 'create':
batch_mapping.update({
value[self.zd_object][i]['external_id']:
value[self.zd_object][i]['external_id']
})
item = value[self.zd_object][i]
item.update({'_log': result[i]})
batch_error[self.zd_object].append(item)
else:
if self.upsert:
mapping = self._get_mapping()
update_batch = {
self.zd_object: [
item for item in batch[self.zd_object]
if item.get('external_id', '') in mapping.keys()
]
}
create_batch = {
self.zd_object: [
item for item in batch[self.zd_object] if item.get(
'external_id', '') not in mapping.keys()
]
}
batch_operation = {
'update': update_batch,
'create': create_batch
}
else:
batch_operation = {'create': batch}
for key, value in batch_operation.items():
result = None
if len(value[self.zd_object]):
if key == 'update':
result = self._update_zd_obj(value)
else:
result = self._create_zd_obj(value)
if result:
for i in range(len(value[self.zd_object])):
id = result[i].get('id', None)
if id:
if self.create_mapping == True:
batch_mapping.update({
value[self.zd_object][i]['external_id']:
id
})
self.log.info(
f"[{self.zd_object}][{key}] SalesForce Id: {value[self.zd_object][i]['external_id']}\tZendesk Id:{id}"
)
else:
if key == 'update':
batch_mapping.update({
value[self.zd_object][i]['external_id']:
value[self.zd_object][i]['external_id']
})
item = value[self.zd_object][i]
item.update({'_log': result[i]})
batch_error[self.zd_object].append(item)
batch_failed = len(batch_error[self.zd_object])
batch_success -= batch_failed
if batch_success > 0:
obj_mapping.update(batch_mapping)
json.dump(batch_mapping, open(batch_mapping_file, 'w+'))
if batch_failed > 0:
obj_error[self.zd_object].extend(batch_error[self.zd_object])
json.dump(batch_error, open(batch_error_file, 'w+'))
self.log.info(f'\tBatch completed.')
self.log.info(f'\t\tSuccess:\t {batch_success}')
self.log.info(
f'\t\tFailed :\t {batch_failed}. Check Error Log file.')
self.log.info(f'All batches completed.')
if self.create_mapping == True:
json.dump(obj_mapping, open(self.id_mapping_file, 'w+'))
json.dump(obj_error, open(self.errors_file, 'w+'))
def _corrupt(fname, index, size):
get_logger().warning("corrupting disk {}".format(index))
error_fpath = r6.get_real_name(index, fname)
error_content = os.urandom(size)
utils.write_content(error_fpath, error_content)
import os
import log_helper
import random
import csv
import json
import re
import datetime
from gdrive.upload import upload_file
logger = log_helper.get_logger('Helpers')
def tagify(text):
if text:
text = "".join([
c for c in text.lower()
if c in 'abcdefghijklmnopqrstuvwyxz -/0123456789.'
])
text = text.replace(' ',
'_').replace('-',
'_').replace('__',
'_').replace('__', '_')
return text
return ''
def date_from_str(date_str):
return datetime.datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S.%f%z')
def add_seconds_to_date(date, seconds):
def logger(mes):
log_helper.get_logger(__meslog__path).info(mes)
import os
from gdrive import service
from log_helper import get_logger
from googleapiclient.http import MediaFileUpload
logger = get_logger('GoogleDrive Service')
drive_service = service.drive_service()
def upload_file(file_name):
link_file = f'{file_name}.webViewLink'
if not os.path.exists(link_file):
media = MediaFileUpload(file_name, resumable=True)
if media.size() > 0:
request = drive_service.files().create(
fields='id,webViewLink',
body={
'name': file_name.split('/')[-1],
'parents': ['1twO1fPD1gCC7MgH3WnlMweX8BOtDUMke']
},
media_body=media)
response = None
logger.info(f'Uploading {file_name}')
while response is None:
status, response = request.next_chunk()
if status:
logger.info("Uploaded %d%%." %
