def _sort_gen(self): gen_list = [] for i, generator in enumerate(self.generator): gen_list.append((generator, i)) sorted_gen_index = sorted(gen_list, key=lambda ele: ele[0]) self.sorted_index = [ele[1] for ele in sorted_gen_index] get_logger().info(self.sorted_index)
def gen_q(data_ndarray, ndim): """ generate q using GF^8 :param data_ndarray: the data ndarray :param ndim: real dim :return: q_ndarray with shape=(1, byte_ndarray.shape[1]) """ transposed = np.transpose(data_ndarray) # print(data_ndarray.shape) get_logger().info('transposed\n{}'.format(transposed)) gf = GF() q_list = [] for _1darray in transposed: bv_list = [] for i, arr_val in enumerate(_1darray): res_i = gf.multiply(gf.generator[i % gf.circle], int(arr_val)) # print('i={}, arr_val={}, res_i={}'.format(i, arr_val, res_i)) bv_list.append(res_i) # map(lambda i: print(i), bv_list) q_value = reduce(operator.xor, bv_list) q_list.append(q_value) arr = np.array(q_list, ndmin=ndim, dtype=config.BYTE_TYPE) get_logger().info("arr={}".format(arr)) # assert arr.shape[1] == data_ndarray.shape[1] return arr
def __gen_raid_array(self, byte_ndarray): """generate full ndarray from raw data ndarray """ # calculate parity and append parity = utils.gen_p(byte_ndarray, ndim=2) write_array = np.concatenate([byte_ndarray, parity]) get_logger().info('write_array=\n{}'.format(write_array)) return write_array
def dump_bitvector(bv, display_base='x', display_width=2): """ helper method to dump bitvector :param bv: :param display_base: :param display_width: :return: """ int_val = int(str(bv), base=2) get_logger().info('{:0{width}{base}}'.format(int_val, base=display_base, width=display_width))
def test_from_content(r6): get_logger().warning("testing from content") original_content = b'good_morning\x03_sir_yes_great\x01\x02' # original_content = b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13' data_fname = 'my.dat' # r6.write(original_content, data_fname) # r6.recover_d_or_p(data_fname, error_index) # r6.recover_d_p(data_fname, 1) # r6.recover_2d(data_fname, 0, 1) # r6_content = r6.read(data_fname, len(original_content)) # assert r6_content == original_content r6.detect_corruption(data_fname)
def log_generator(self, result): """ log_g operation :param result: the value to be log :return: int result """ get_logger().info(type(result)) assert type(result) in (int, BitVector) if isinstance(result, BitVector): result = result.int_val() for i in range(self.circle): if self.generator[i].int_val() == result: return i
def swap(self, byte_ndarray): """ swap the data from the final array to the destination slot :param byte_ndarray: :return: """ # get_logger().info('before\n{}'.format(byte_ndarray)) for i in xrange(self.N - 1): j = self.get_parity_index(i) get_logger().info('{}, {}'.format(byte_ndarray[i][j], byte_ndarray[self.N - 1][j])) byte_ndarray[i][j], byte_ndarray[self.N - 1][j] = byte_ndarray[self.N - 1][j], byte_ndarray[i][j] # get_logger().info('after\n{}'.format(byte_ndarray)) return byte_ndarray
def swap(self, byte_ndarray): """ swap the data from the final array to the destination slot :param byte_ndarray: :return: """ # get_logger().info('before\n{}'.format(byte_ndarray)) for i in xrange(self.N - 1): j = self.get_parity_index(i) get_logger().info('{}, {}'.format(byte_ndarray[i][j], byte_ndarray[self.N - 1][j])) byte_ndarray[i][j], byte_ndarray[self.N - 1][j] = byte_ndarray[ self.N - 1][j], byte_ndarray[i][j] # get_logger().info('after\n{}'.format(byte_ndarray)) return byte_ndarray
def init_generator(self): """ pre-compute the lookup table of the generator """ get_logger().info('init generator') self.generator = [] res = BitVector(intVal=1) base = BitVector(intVal=2) counter = 0 while True: self.generator.append(res) self.dump_bitvector(res) counter += 1 if counter >= self.circle: break res = self.multiply(res, base) assert len(set(self.generator)) == self.circle
def init_generator(self): """ generate look up generator table """ self.generator = [] counter = 0 res = 1 base = 2 while True: self.generator.append(res) counter += 1 if counter >= self.circle: break res = self.multiply(res, base) assert len(self.generator) == self.circle self._sort_gen() for g in self.generator: get_logger().info('{:0{width}{base}}'.format(g, base='x', width=2))
def test_from_data_file(r6): import driver def _corrupt(fname, index, size): get_logger().warning("corrupting disk {}".format(index)) error_fpath = r6.get_real_name(index, fname) error_content = os.urandom(size) utils.write_content(error_fpath, error_content) def _corrupt2(fname, indexes, size): for index in indexes: _corrupt(fname, index, size) data_fname = 'data3' SIZE = 32768 driver.gen_rnd_file(data_fname, SIZE, 'text') fpath = os.path.join(config.root, 'data3') original_content = utils.read_content(fpath) r6.write(original_content, data_fname) r6.detect_corruption(data_fname) for error_index in [0, 3, r6.N - 2, r6.N - 1]: error_size = SIZE / 13 _corrupt(data_fname, error_index, error_size) found_error_index = r6.detect_corruption(data_fname) if found_error_index is not None: get_logger().warning("recover disk {}".format(error_index)) assert found_error_index == error_index if found_error_index < r6.N - 1: r6.recover_d_or_p(data_fname, found_error_index) else: r6.recover_q(data_fname) r6.detect_corruption(data_fname) ##################################################### get_logger().warning("testing recover_d_q") error_indexes = [4, r6.N - 1] size = SIZE / (r6.N - 4) _corrupt2(data_fname, error_indexes, size) r6.recover_d_q(data_fname, error_indexes[0]) r6.detect_corruption(data_fname) ##################################################### get_logger().warning("testing recover_2d") error_indexes = [0, 1] size = SIZE / (r6.N + 2) _corrupt2(data_fname, error_indexes, size) r6.recover_2d(data_fname, error_indexes[0], error_indexes[1]) r6.detect_corruption(data_fname) ##################################################### get_logger().warning("testing recover_d_p") error_indexes = [0, r6.N - 2] size = SIZE / (r6.N - 2) _corrupt2(data_fname, error_indexes, size) r6.recover_d_p(data_fname, error_indexes[0]) r6.detect_corruption(data_fname)
def gen_rnd_file(fname, size, content_type): """ randomly generate data chunk :param fname: the name denoting the data :param size: the size of the data chunk :param content_type: can be 'text' or others; if it's 'text', make the chunk only contain ascii letters :return: """ if content_type == 'text': # noinspection PyUnusedLocal content = ''.join([random.choice(string.ascii_letters) for i in xrange(size)]) else: content = os.urandom(size) fpath = os.path.join(config.root, fname) if os.path.isfile(fpath): file_size = os.stat(fpath).st_size if file_size == size: get_logger().warning('fname={} with size={} exists'.format(fname, size)) return get_logger().warning('generating fname={} with size={}'.format(fname, size)) utils.write_content(fpath, content)
def _gen_ndarray_from_content(self, content, num): """ convert the content into numpy 2-darray :param content: read content :param num: 1st dimension size :return: """ # gen N-1 length empty list content_list = [[] for i in repeat(None, num)] for i in xrange(len(content)): mod_i = i % num content_list[mod_i].append(content[i]) byte_list = [] length = len(sorted(content_list, key=len, reverse=True)[0]) # fill 0 for content in content_list: current_str_list = [ord(s) for s in content] + [self.ZERO] * (length - len(content)) byte_list.append(current_str_list) byte_nparray = np.array(byte_list, dtype=self.BYTE_TYPE) get_logger().info('byte_array'.format(byte_nparray)) return byte_nparray
def _gen_ndarray_from_content(self, content, num): """ convert the content into numpy 2-darray :param content: read content :param num: 1st dimension size :return: """ # gen N-1 length empty list content_list = [[] for i in repeat(None, num)] for i in xrange(len(content)): mod_i = i % num content_list[mod_i].append(content[i]) byte_list = [] length = len(sorted(content_list, key=len, reverse=True)[0]) # fill 0 for content in content_list: current_str_list = [ord(s) for s in content ] + [self.ZERO] * (length - len(content)) byte_list.append(current_str_list) byte_nparray = np.array(byte_list, dtype=self.BYTE_TYPE) get_logger().info('byte_array'.format(byte_nparray)) return byte_nparray
def gen_rnd_file(fname, size, content_type): """ randomly generate data chunk :param fname: the name denoting the data :param size: the size of the data chunk :param content_type: can be 'text' or others; if it's 'text', make the chunk only contain ascii letters :return: """ if content_type == 'text': # noinspection PyUnusedLocal content = ''.join( [random.choice(string.ascii_letters) for i in xrange(size)]) else: content = os.urandom(size) fpath = os.path.join(config.root, fname) if os.path.isfile(fpath): file_size = os.stat(fpath).st_size if file_size == size: get_logger().warning('fname={} with size={} exists'.format( fname, size)) return get_logger().warning('generating fname={} with size={}'.format( fname, size)) utils.write_content(fpath, content)
def duration(then): t = time.time() - then get_logger().info('takes {}'.format(t)) return t
'--name', help='container name regex pattern', dest='name', required=True) parser.add_argument( '-s', '--sleep', type=int, default=30, help='sleep duration in seconds between consecutive actions', dest='sleep') parser.add_argument("-v", "--verbose", help="increase output verbosity", action="store_true") logger = log_helper.get_logger() def signal_handler(signal, frame): """ handle the Ctrl-C keyboard interrupt by cleanly exiting """ logger.debug("Exiting") sys.exit(0) signal.signal(signal.SIGINT, signal_handler) def filter_containers_by_name(pattern): """
#!/usr/bin/env python import yaml from log_helper import get_logger l = get_logger('config') #this class is the parser for yml file def parser(): filename = "../settings.yml" with open(filename, 'r') as stream: try: l.info('Parsing Config Files') l.info('Opening {}'.format(filename)) l.info('{} Successfully Read'.format(filename)) val = yaml.load(stream) # print(val) except yaml.YAMLError as exc: l.fatal('Error parsing file {filename}: {exc}. Please fix and try ' 'again.'.format(filename=filename, exc=exc)) # print(exc) return val def ethosUrl():
def logger(msg): global _log_path log_helper.get_logger(_log_path).info(msg)
class SalesForce(): log = get_logger('SalesForce') __auth_params__ = set( ['grant_type', 'client_id', 'client_secret', 'username', 'password']) _service_path = '/services/data/v40.0' def __init__(self, **kwargs): auth_params = {} if self.__auth_params__.issubset(set(kwargs.keys())): for param in self.__auth_params__: auth_params.update({param: kwargs.get(param, None)}) self.session = requests.Session() self.session.headers.update({ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36' }) r = self.session.post( "https://login.salesforce.com/services/oauth2/token", params=auth_params) self._access_token = r.json().get("access_token", None) self.session.headers[ 'Authorization'] = f'Bearer {self._access_token}' self.session.cookies.set(name='sid', value=kwargs['cookie_sid'], domain=kwargs['cookie_domain']) self.session.cookies.set( name='atlassian.xsrf.token', value= 'B2YH-NCM1-ID0V-D60M_8284e4e214fc14e59ad9ea5bbff670c9b3d20255_lin', domain='jira.devfactory.com') self.session.cookies.set( name='JSESSIONID', value='E70BC00650914C5DE7986E27DFCE0F47.jira-prod-node-4', domain='jira.devfactory.com') self.session.cookies.set(name='route', value='1574861615.812.134449.885484', domain='jira.devfactory.com') self.instance_url = r.json().get("instance_url", None) def get_image(self, url): response = self.session.get(url, timeout=(2, 25)) return response.content def _api_call(self, action, parameters={}, method='get', data={}): """ Helper function to make calls to Salesforce REST API. Parameters: action (the URL), URL params, method (get, post or patch), data for POST/PATCH. """ headers = { 'Content-type': 'application/json', 'Accept-Encoding': 'gzip', 'Authorization': 'Bearer %s' % self._access_token } if self._service_path not in action: action = self._service_path + action if method == 'get': r = self.session.request(method, self.instance_url + action, headers=headers, params=parameters) elif method in ['post', 'patch']: r = self.session.request(method, self.instance_url + action, headers=headers, json=data, params=parameters) else: # other methods not implemented in this example raise ValueError('Method should be get or post or patch.') self.log.debug('Debug: API %s call: %s' % (method, r.url)) if r.status_code < 300: if method == 'patch': return None else: if 'application/json' in r.headers.get('content-type'): return r.json() else: return r.content else: raise Exception('API error when calling %s : %s' % (r.url, r.content)) def discover_fields(self, sf_object): response = self._api_call(f'/sobjects/{sf_object}/describe') if not response.get('queryable', False): raise Exception("This object is not queryable") return response.get('fields', None) def get_case_comments(self, case_id): fields = ['CreatedById', 'CreatedDate', 'CommentBody', 'IsPublished'] conditions = [f'ParentId={case_id}'] response = self.query_all('CaseComments', fields, conditions) return response def get_case_attachments(self, case_id): response = self._api_call(f'/Case/{case_id}/Attachments') return response def download_attachment(self, file_id): response = self._api_call(f'/sobjects/Attachment/{file_id}/body') return response def query_all(self, sf_object, fields, conditions=[], limit=None): query_str = f"SELECT {', '.join(fields)} FROM {sf_object} " + ( "WHERE " + " AND ".join(conditions) if len(conditions) > 0 else "") + (f"LIMIT {limit}" if limit else "") response = self._api_call('/queryAll/', {'q': query_str}) next = response.get('nextRecordsUrl', None) yield response.get('records', []) while next: response = self._api_call(next) next = response.get('nextRecordsUrl', None) yield response.get('records', [])
import json import csv import os from config import ENV from salesforce import SalesForce from zendesk import Zendesk from migration import MigrationItem from log_helper import get_logger import helpers log = get_logger('Main') if __name__ == "__main__": migration_plan = json.load(open('migration_plan.json', 'r')) sf_config = migration_plan['salesforce'] sf = SalesForce(**sf_config) zd_config = migration_plan['zendesk'] zd = Zendesk(**zd_config) for item in migration_plan['migration_items']: if item['skip'] == False: if item['type'] == 'migration_object': migration_item = MigrationItem(sf, zd, mode=ENV, **item) if migration_item.skip == False: try: migration_item.migrate() except Exception as err: log.critical(err) raise err elif item['type'] == 'script':
#!/usr/bin/env python from pyEthOS import pyEthOS as ethos import requests import json from config import ethosUrl from log_helper import get_logger l = get_logger('ethosapi') def ethosapi(): PANEL_NAME = ethosUrl() DEBUG = False #api = ethos.EthOS_API(PANEL_NAME, debug=DEBUG) try: api = ethos.EthOS_API(PANEL_NAME, debug=DEBUG) except requests.exceptions.RequestException as e: print(e) sys.exit(1) #req = requests.get('http://altaiu.ethosdistro.com') #req2 = req.json() apiReq = api.get_summary()
def logger(mes): log_helper.get_logger(__log__path).info(mes)
def _corrupt(fname, index, size): get_logger().warning("corrupting disk {}".format(index)) error_fpath = r6.get_real_name(index, fname) error_content = os.urandom(size) utils.write_content(error_fpath, error_content)
class RAID6(RAID): _logger = get_logger() def __init__(self, N): assert 4 <= N super(RAID6, self).__init__(N) # gf object self.gf = GF() def check(self, byte_ndarray): """ check involves p and q :param byte_ndarray: all ndarray including p and q :return: """ # check p data_p_ndarray = byte_ndarray[:-1] utils.check_data_p(data_p_ndarray) # check_q data_ndarray = byte_ndarray[:-2] q_ndarray = byte_ndarray[-1:] utils.check_q(data_ndarray, q_ndarray) def read(self, fname, size): """ read size chunk from fname(RAID6 system) """ byte_ndarray = self._read_n(fname, self.N) self.check(byte_ndarray) data_ndarray = byte_ndarray[:-2] flat_list = data_ndarray.ravel(1)[:size] flat_str_list = [chr(e) for e in flat_list] return ''.join(flat_str_list) def recover(self, fname, exclude): """ since there are different cases, we raise an error indicating it should not be called """ raise NotImplementedError("not implemented; split into several cases") def _get_corrupted_data_disk(self, P_star, Q_star): """ from P_star and Q_star, find the corrupted data disk index procondition: P_star!={00}, Q_star!={00} (in vector sense) """ p0 = int(P_star[0][0]) q0 = int(Q_star[0][0]) log_p0 = self.gf.log_generator(p0) log_q0 = self.gf.log_generator(q0) return (log_q0 - log_p0) % self.gf.circle def detect_corruption(self, fname): """ single disk corruption detection :param fname: data name in RAID6 system :return: corrupted disk index; for p, self.N-2; for q, self.N-1 """ # all disks, including P, Q byte_ndarray = self._read_n(fname, self.N) data_ndarray = byte_ndarray[:-2] self._logger.info("byte_ndarray=\n{}".format(byte_ndarray)) P = byte_ndarray[-2:-1] self._logger.info("p={}".format(P)) Q = byte_ndarray[-1] self._logger.info("Q={}".format(Q)) P_prime = utils.gen_p(data_ndarray, ndim=2) self._logger.info("P_prime={}".format(P_prime)) Q_prime = utils.gen_q(data_ndarray, ndim=2) self._logger.info("Q_prime={}".format(Q_prime)) P_star = np.bitwise_xor(P, P_prime) Q_star = np.bitwise_xor(Q, Q_prime) P_nonzero = np.count_nonzero(P_star) Q_nonzero = np.count_nonzero(Q_star) if P_nonzero == 0 and Q_nonzero == 0: print("no corruption") return None elif P_nonzero == 0 and Q_nonzero != 0: print("Q corruption") return self.N - 1 elif P_nonzero != 0 and Q_nonzero == 0: print("P corruption") return self.N - 2 else: index = self._get_corrupted_data_disk(P_star, Q_star) print("data disk {} corruption".format(index)) return index def recover_d_or_p(self, fname, index): """ recover data drive or 'p' drive, simply using XOR :param fname: data name :param index: data disk or p disk index :return: """ assert 0 <= index < self.N - 1 byte_ndarray = self._read_n(fname, self.N - 1, exclude=index) parity = utils.gen_p(byte_ndarray, ndim=1) content = self._1darray_to_str(parity) fpath = self.get_real_name(index, fname) utils.write_content(fpath, content) # check data or p read_ndarray = self._read_n(fname, self.N - 1) utils.check_data_p(read_ndarray) def recover_q(self, fname): """ recover 'q' drive, recompute :param fname: data name :return: """ byte_ndarray = self._read_n(fname, self.N - 2) q_ndarray = utils.gen_q(byte_ndarray, ndim=2) assert q_ndarray.ndim == 2 new_num = q_ndarray.shape[1] q_ndarray.shape = (new_num, ) content = self._1darray_to_str(q_ndarray) fpath = self.get_real_name(self.N - 1, fname) utils.write_content(fpath, content) def recover_d_q(self, fname, index): """ recover data/'p' drive (index) and 'q' drive: firstly using XOR to recover data drive, then recompute q :param fname: data name :param index: corrupted data disk index :return: """ self.recover_d_or_p(fname, index) self.recover_q(fname) def recover_2d(self, fname, x, y): """ recover data drives (x and y) :param fname: data name :param x: corrupted data disk index :param y: corrupted data disk index :return: """ assert 0 <= x < self.N - 2 assert 0 <= y < self.N - 2 assert x != y byte_ndarray = self._read_n(fname, self.N, exclude=[x, y]) DD = byte_ndarray[:-2] P = byte_ndarray[-2:-1] Q = byte_ndarray[-1:] # Pxy Pxy = utils.gen_p(DD, ndim=2) # Qxy Qxy = utils.gen_q(DD, ndim=2) # Axy, Bxy A = self.gf.Axy(x, y) B = self.gf.Bxy(x, y) # Dx first = utils.gf_a_multiply_list(A, utils.gf_1darray_add(P, Pxy)) second = utils.gf_a_multiply_list(B, utils.gf_1darray_add(Q, Qxy)) Dx = utils.gf_1darray_add(np.array(first, dtype=config.BYTE_TYPE), np.array(second, dtype=config.BYTE_TYPE)) Dx_content = self._1darray_to_str(Dx) x_fpath = self.get_real_name(x, fname) utils.write_content(x_fpath, Dx_content) # Dy Dy = utils.gf_1darray_add(P ^ Pxy, Dx) Dy_content = self._1darray_to_str(Dy) y_fpath = self.get_real_name(y, fname) utils.write_content(y_fpath, Dy_content) def recover_d_p(self, fname, index): """ recover data drive (index) and 'p' drive :param fname: data name :param index: data disk index :return: """ assert 0 <= index < self.N - 2 byte_ndarray = self._read_n(fname, self.N, exclude=index) DD = byte_ndarray[:-2] Q = byte_ndarray[-1:] # Dx Qx = utils.gen_q(DD, ndim=2) g_x_inv = self.gf.generator[(self.gf.circle - index) % self.gf.circle] ### _add_list = utils.gf_1darray_add(Q, Qx) Dx_list = utils.gf_a_multiply_list(g_x_inv, _add_list) ### Dx_content = ''.join(chr(i) for i in Dx_list) x_fpath = self.get_real_name(index, fname) utils.write_content(x_fpath, Dx_content) # p Dx = np.array(Dx_list, ndmin=2) assert Dx.shape[1] == byte_ndarray.shape[1] # update firstly DD[index] = Dx P = utils.gen_p(DD, ndim=1) assert P.shape[0] == byte_ndarray.shape[1] # do not need to update DD P_content = self._1darray_to_str(P) P_path = self.get_real_name(self.N - 2, fname) utils.write_content(P_path, P_content) def write(self, content, fname): """ write content to fname(in RAID6 system) """ byte_ndarray = self._gen_ndarray_from_content(content, self.N - 2) p_ndarray = utils.gen_p(byte_ndarray, ndim=2) q_ndarray = utils.gen_q(byte_ndarray, ndim=2) write_ndarray = np.concatenate([byte_ndarray, p_ndarray, q_ndarray]) self._write_n(fname, write_ndarray, self.N)
class MigrationItem(): log = get_logger('MigrationPlan') __data_folder__ = 'migration_data' def __init__(self, sf, zd, **kwargs): self.env_mode = kwargs.get('mode', 'dev') self._sf = sf self._zd = zd self.zd_custom_endpoint = kwargs.get('zd_custom_endpoint', None) self.skip = kwargs.get('skip', True) self.sf_object = kwargs.get('sf_object', None) self.payload_file = kwargs.get('file', None) self.zd_object = kwargs['zd_object'] self.limit = kwargs.get('limit', None) self.upsert = kwargs.get('upsert', True) self.create_mapping = kwargs.get('create_mapping', True) self.after_download = kwargs.get('after_download', []) self.bulk_create_or_update = kwargs.get('bulk_create_or_update', False) self.sf_fields = [ field for field in kwargs.get('sf_fields', []) if type(field) == str ] self.sf_join_fields = [ field for field in kwargs.get('sf_fields', []) if type(field) == dict ] if len(self.sf_fields) == 0 and self.sf_object: self.sf_fields = [ field['name'] for field in sf.discover_fields(self.sf_object) ] self.sf_conditions = kwargs.get('sf_conditions', []) self.fields_mapping = kwargs.get('fields_mapping', []) self.force_download = kwargs.get('force_download', True) self.batch_folder = f'{self.__data_folder__}/{self.sf_object or self.zd_object}/batches' self.export_file = f'{self.__data_folder__}/{self.sf_object}/data.csv' self.data_file = f'{self.__data_folder__}/{self.sf_object}/data.json' self.id_mapping_file = f'{self.__data_folder__}/{self.sf_object}/{self.env_mode}-mapping.json' self.errors_file = f'{self.__data_folder__}/{self.sf_object or self.zd_object}/errors.json' self.mapping_errors_file = f'{self.__data_folder__}/{self.sf_object or self.zd_object}/mapping-errors.json' def log_mapping_error(self, obj, source, key, value): json_payload = [{ 'object': obj, 'source': source, 'key': key, 'value': value }] if not os.path.exists(self.mapping_errors_file): json.dump(json_payload, open(self.mapping_errors_file, 'w+')) else: log_json = json.load(open(self.mapping_errors_file, 'r')) log_json.extend(json_payload) json.dump(log_json, open(self.mapping_errors_file, 'w+')) def __build_directory(self): os.makedirs( f'{self.__data_folder__}/{self.sf_object or self.zd_object}/batches', exist_ok=True) def __parse_sf_row(self, row, fields=None): parsed_row = {} for field in (fields or self.sf_fields): if len(field.split('.')) == 1: value = row[field] elif row[field.split('.')[0]] is not None: field_key = field.split('.')[0] child_key = field.split('.')[-1] value = row[field_key].get(child_key, None) else: value = None parsed_row.update({field: value}) return parsed_row def get_data(self): if self.sf_object is None and self.payload_file: data = json.load(open(self.payload_file, 'r', encoding='utf-8-sig')) else: if os.path.exists(self.data_file) is False: self.download_data() data = json.load(open(self.data_file, 'r', encoding='utf-8-sig')) return data def download_data(self): self.__build_directory() self.log.info( f'Downloading data from SalesForce for {self.sf_object} object') csv_writer = csv.DictWriter(open(self.export_file, 'w+', newline='', encoding="utf-8-sig"), self.sf_fields, extrasaction='ignore') csv_writer.writeheader() data = [] for rows in self._sf.query_all( self.sf_object, [field for field in self.sf_fields if type(field) == str], self.sf_conditions, limit=self.limit): if len(rows) > 0: for row in rows: parsed_row = self.__parse_sf_row(row) for join_field in self.sf_join_fields: self.log.info( f'Getting {join_field["sf_object"]} for {self.sf_object}:{row["Id"]} object' ) parsed_row.update({join_field['sf_object']: []}) for join_rows in self._sf.query_all( join_field['sf_object'], join_field['sf_fields'], [ eval(condition, {'row': row}) for condition in join_field['sf_conditions'] ]): for join_row in join_rows: parsed_row[join_field['sf_object']].append( self.__parse_sf_row( join_row, join_field['sf_fields'])) data.append(parsed_row) csv_writer.writerow(parsed_row) json.dump(data, open(self.data_file, 'w+', encoding='utf-8-sig')) self.log.info('Done.') def on_after_download(self): for func in self.after_download: self.log.info(f'Evaluating function after download {func}') eval(func) def _update_zd_obj(self, json): result = self._zd.put( f'/{self.zd_custom_endpoint or self.zd_object}/update_many.json', json=json, response_container='job_status') status = result['status'] while status in ['queued', 'working']: job_url = result['url'] result = self._zd.get(job_url, 'job_status') status = result['status'] if status in ['queued', 'working']: time.sleep(10) return result.get('results', None) def _create_zd_obj(self, json): result = None while result is None: result = self._zd.post( f'/{self.zd_custom_endpoint or self.zd_object}/create_many.json', data=json, response_container='job_status') if 'status_code' in result and result.status_code == 429: wait_time = int(result.headers['Retry-After']) result = None time.sleep(wait_time) continue status = result['status'] while status in ['queued', 'working']: job_url = result['url'] result = self._zd.get(job_url, 'job_status') status = result['status'] if status in ['queued', 'working']: message = result.get('message', None) if message: self.log.info(message) else: self.log.info('Waiting reponse...') time.sleep(10) return result.get('results', None) def _create_or_update_zd_obj(self, json): result = self._zd.post( f'/{self.zd_custom_endpoint or self.zd_object}/create_or_update_many.json', data=json, response_container='job_status') status = result['status'] if 'status' in result else 'error' while status in ['queued', 'working', 'error']: job_url = result['url'] result = self._zd.get(job_url, 'job_status') status = result['status'] if status in ['queued', 'working']: self.log.info('Waiting reponse...') time.sleep(10) return result.get('results', None) def get_zd_payload(self): self.log.info(f'Building Zendesk {self.zd_object} object payload') data = self.get_data() total = len(data) if self.sf_object: if os.path.exists( os.path.join(self.__data_folder__, self.sf_object, 'payload.json')): return json.load( open( os.path.join(self.__data_folder__, self.sf_object, 'payload.json'), 'r')) payload = {self.zd_object: []} for idx, fields in enumerate(data): self.log.info(f'Processing... {idx}/{total}') mapped = {} for map_item in self.fields_mapping: field_type = map_item['field']['type'] field_key = map_item['field']['key'] field_value = map_item['value'] value_type = map_item.get('type', None) if value_type == 'sf_field': field_value = eval(field_value, { "helpers": helpers, "self": self }, { key.replace('.', '_'): value for key, value in fields.items() }) elif value_type == 'mapping': mapping_source = map_item.get('source', None) mapping_key = field_value if mapping_source and mapping_key: mapping = self._get_mapping(mapping_source) if mapping: field_value = mapping.get( fields[mapping_key], None) if field_value is None: self.log_mapping_error( fields, mapping_source, mapping_key, fields[mapping_key]) self.log.warning( f'No Mapping found for {field_key} in {mapping_source} |\tkey:{fields[mapping_key]}' ) else: fallback_value = map_item.get( 'fallback_value', None) field_value = fallback_value self.log_mapping_error(fields, mapping_source, mapping_key, fields[mapping_key]) self.log.warning( f'No Mapping found for {field_key} in {mapping_source}. Fallbak value is used instead' ) else: field_value = map_item['value'] if field_value is not None: if field_type == 'standard': mapped.update({field_key: field_value}) else: if mapped.get(field_type, None): mapped[field_type].update( {field_key: field_value}) else: mapped.update( {field_type: { field_key: field_value }}) payload[self.zd_object].append(mapped) self.log.info(f'Done.') json.dump( payload, open( os.path.join(self.__data_folder__, self.sf_object, 'payload.json'), 'w+')) return payload return data def _create_batch_payload(self): batches = [] payload = self.get_zd_payload() slices = round(len(payload[self.zd_object]) / 100 + 0.5) slice_size = 100 for i in range(slices): batches.append({ self.zd_object: payload[self.zd_object][i * slice_size:(i + 1) * slice_size] }) return batches def _create_batch_list(self, items): batches = [] slices = round(len(items) / 100 + 0.5) slice_size = 100 for i in range(slices): batches.append( list(items)[i * slice_size:slice_size + i * slice_size]) return batches def _get_mapping(self, source=None): if source: file = f'{self.__data_folder__}/{source}/{self.env_mode}-mapping.json' return self.__get_json_file(file) if os.path.exists(self.id_mapping_file): return self.__get_json_file(self.id_mapping_file) return {} def __get_json_file(self, file): if os.path.exists(file): f = open(file, 'r') data = f.read() f.close() return json.loads(data) return None def get_sync_diff(self): mapping = self._get_mapping() if mapping: data = self.get_data() new_obj_ids = set([item['Id'] for item in data]) old_obj_ids = set([key for key in mapping.keys()]) diff = old_obj_ids.difference(new_obj_ids) return diff return None def migrate(self): self.__build_directory() self.log.info(f'Starting Migration of {self.zd_object} Object') import_payload_file = f'migration_data/{self.sf_object}/import_payload.json' if self.sf_object: if self.force_download == True: self.download_data() self.on_after_download() if self.payload_file: payload_batches = self._create_batch_payload() else: if os.path.exists(import_payload_file): payload_batches = json.load( open(import_payload_file, 'r', encoding='utf-8-sig')) else: payload_batches = self._create_batch_payload() json.dump( payload_batches, open(import_payload_file, 'w+', encoding='utf-8-sig')) obj_mapping = {} obj_error = {self.zd_object: []} self.log.info(f'Uploading data to zendesk') for idx, batch in enumerate(payload_batches): batch_mapping = {} batch_error = {self.zd_object: []} batch_mapping_file = os.path.join(self.batch_folder, f'batch_success_{idx}.json') batch_error_file = os.path.join(self.batch_folder, f'batch_error_{idx}.json') if os.path.exists(batch_mapping_file): self.log.info(f'Skipped batch {idx}. Already processed.') obj_mapping.update( json.load( open(batch_mapping_file, 'r', encoding='utf-8-sig'))) if os.path.exists(batch_error_file): obj_error[self.zd_object].extend( json.load( open(batch_error_file, 'r', encoding='utf-8-sig'))) continue batch_success = len(batch[self.zd_object]) self.log.info(f'\tBatch {idx} of {len(payload_batches)}') json.dump({}, open(batch_mapping_file, 'w+')) if self.bulk_create_or_update == True: value = batch key = 'bulk_create_update' result = self._create_or_update_zd_obj(batch) if result: for i in range(len(value[self.zd_object])): id = result[i].get('id', None) if id: if self.create_mapping == True: batch_mapping.update({ value[self.zd_object][i]['external_id']: id }) self.log.info( f"[{self.zd_object}][{key}] SalesForce Id: {value[self.zd_object][i]['external_id']}\tZendesk Id:{id}" ) else: if key != 'create': batch_mapping.update({ value[self.zd_object][i]['external_id']: value[self.zd_object][i]['external_id'] }) item = value[self.zd_object][i] item.update({'_log': result[i]}) batch_error[self.zd_object].append(item) else: if self.upsert: mapping = self._get_mapping() update_batch = { self.zd_object: [ item for item in batch[self.zd_object] if item.get('external_id', '') in mapping.keys() ] } create_batch = { self.zd_object: [ item for item in batch[self.zd_object] if item.get( 'external_id', '') not in mapping.keys() ] } batch_operation = { 'update': update_batch, 'create': create_batch } else: batch_operation = {'create': batch} for key, value in batch_operation.items(): result = None if len(value[self.zd_object]): if key == 'update': result = self._update_zd_obj(value) else: result = self._create_zd_obj(value) if result: for i in range(len(value[self.zd_object])): id = result[i].get('id', None) if id: if self.create_mapping == True: batch_mapping.update({ value[self.zd_object][i]['external_id']: id }) self.log.info( f"[{self.zd_object}][{key}] SalesForce Id: {value[self.zd_object][i]['external_id']}\tZendesk Id:{id}" ) else: if key == 'update': batch_mapping.update({ value[self.zd_object][i]['external_id']: value[self.zd_object][i]['external_id'] }) item = value[self.zd_object][i] item.update({'_log': result[i]}) batch_error[self.zd_object].append(item) batch_failed = len(batch_error[self.zd_object]) batch_success -= batch_failed if batch_success > 0: obj_mapping.update(batch_mapping) json.dump(batch_mapping, open(batch_mapping_file, 'w+')) if batch_failed > 0: obj_error[self.zd_object].extend(batch_error[self.zd_object]) json.dump(batch_error, open(batch_error_file, 'w+')) self.log.info(f'\tBatch completed.') self.log.info(f'\t\tSuccess:\t {batch_success}') self.log.info( f'\t\tFailed :\t {batch_failed}. Check Error Log file.') self.log.info(f'All batches completed.') if self.create_mapping == True: json.dump(obj_mapping, open(self.id_mapping_file, 'w+')) json.dump(obj_error, open(self.errors_file, 'w+'))
import os import log_helper import random import csv import json import re import datetime from gdrive.upload import upload_file logger = log_helper.get_logger('Helpers') def tagify(text): if text: text = "".join([ c for c in text.lower() if c in 'abcdefghijklmnopqrstuvwyxz -/0123456789.' ]) text = text.replace(' ', '_').replace('-', '_').replace('__', '_').replace('__', '_') return text return '' def date_from_str(date_str): return datetime.datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S.%f%z') def add_seconds_to_date(date, seconds):
def logger(mes): log_helper.get_logger(__meslog__path).info(mes)
import os from gdrive import service from log_helper import get_logger from googleapiclient.http import MediaFileUpload logger = get_logger('GoogleDrive Service') drive_service = service.drive_service() def upload_file(file_name): link_file = f'{file_name}.webViewLink' if not os.path.exists(link_file): media = MediaFileUpload(file_name, resumable=True) if media.size() > 0: request = drive_service.files().create( fields='id,webViewLink', body={ 'name': file_name.split('/')[-1], 'parents': ['1twO1fPD1gCC7MgH3WnlMweX8BOtDUMke'] }, media_body=media) response = None logger.info(f'Uploading {file_name}') while response is None: status, response = request.next_chunk() if status: logger.info("Uploaded %d%%." %