def writePayments(self): # write payments to csv file try: payment.write(self) except: log.writeError(e)
def getEventData(self, startDate, endDate): try: eventURL = 'http://api.meetup.com/2/events?key='+self.getApiKey()+'&time='+self.getTimeStamp(startDate)+','+self.getTimeStamp(endDate)+'&group_urlname='+self.getGroupName()+'&fields=fee&status=past' self._events = json.load(urllib2.urlopen(eventURL))['results'] except Exception, e: log.writeError(e)
def writePayments(self, strCSVFilePath): if self.getPayments() == None: return None try: writer = csv.writer(open(strCSVFilePath, 'w'), quoting=csv.QUOTE_ALL) writer.writerow(["key", "datetime", "description", "amount", "fee"]) payments = self.getPayments() for transaction in payments: row = list() try: row.append(str(transaction['id'])) except: row.append("") try: row.append(str(transaction['created_at'])) except: row.append("") try: row.append(str(transaction['description'])) except: row.append("") try: row.append(str(transaction['net_total_money']['amount'])) except: row.append("") try: row.append(str(transaction['processing_fee_money']['amount']).replace('-', '')) except: row.append("") writer.writerow(row) except Exception, e: log.writeError(e)
def load(jobId, strCSVFilePath): try: db.execute(_sqlLoadPayments(jobId, strCSVFilePath)) return True except Exception, e: log.writeError(e) return False
def setStartDate(self, startDate): if type(startDate) is datetime: self._startDate = startDate else: try: self.startDate = datetime.strptime(startDate, "%Y-%m-%d") except TypeError: log.writeError("StartDate parameter cannot be cast to a datetime object")
def setEndDate(self, endDate): if type(endDate) is datetime: self._endDate = endDate else: try: self._endDate = datetime.strptime(endDate, "%Y-%m-%d") except TypeError: log.writeError("EndDate parameter cannot be cast to a datetime object")
def getDate(Date): try: if type(Date) is datetime: return Date else: return datetime.strptime(str(Date), '%Y-%m-%d') except Exception, e: log.writeError(e)
def getPayments(self, startDate, endDate): # retrieve payment data from Amazon via api try: self.setStartDate(startDate) self.setEndDate(endDate) self._payments = payment.get(self) except Exception, e: log.writeError(e)
def getTimeStamp(Date): try: if type(Date) is datetime: return str(int((datetime(Date.year, Date.month, Date.day) - \ datetime(1970, 1,1)).total_seconds()))+'000' else: return '0' except Exception, e: log.writeError(e)
def write(amazonProvider, strCSVFileName): if amazonProvider.getPaymentResults() == None: log.writeInfo('[AMAZON] There are no payments.') return False try: writer = csv.writer(open(strCSVFileName, 'w'), quoting=csv.QUOTE_ALL) writer.writerow(["key","datetime","fps_operation", "sender_key", "sender_name", "description", "fees", "status", "amount"]) for transaction in amazonProvider.getPaymentResults(): row = list() row.append(str(transaction.TransactionId)) try: row.append(str(transaction.DateCompleted)) except: row.append("") try: row.append(str(transaction.FPSOperation)) except: row.append("") try: row.append(str(transaction.SenderTokenId)) except: row.append("") try: row.append(transaction.SenderName.encode("ascii", "ignore")) except: row.append("") if "Description:" in str(transaction.TransactionPart[0]).split('(')[1].split(',')[0]: row.append(str(transaction.TransactionPart[0]).split('(')[1].split(',')[0].split("'")[1]) else: row.append("") try: row.append(str(transaction.FPSFees)) except: row.append("") try: row.append(str(transaction.TransactionStatus)) except: row.append("") try: row.append(str(transaction.TransactionAmount)) except: row.append("") writer.writerow(row) return True except Exception, e: log.writeError(e) return False
def main(): logging.basicConfig(filename='/home/jleto/AmazonMeetupConnector/log/novalabs-data-import.log', level=logging.INFO) projectProperties = properties.get('data-import.properties') meetupProperties = properties.get('meetup.properties') amazonProperties = properties.get('amazon.properties') squareupProperties = properties.get('squareup.properties') try: log.writeInfo(' [GENERAL] Generating batches.') result = jobProcessor.generate() except Exception, e: log.writeError(e)
def process(job, squareupProperties): squareupProvider = squareup.squareup(squareupProperties['PERSONAL_ACCESS_TOKEN']) squareupProvider.getPaymentData(strDate(job['batch_key'])) if squareupProvider.getPaymentCount() > 0: try: path = squareupProperties['datafile_path'] strCSVFilePath = path + "/squareup-payment-" + strDate(job['batch_key']) + ".csv" log.writeInfo('[SQUARE UP] Writing '+strCSVFilePath+' to disk.') squareupProvider.writePayments(strCSVFilePath) squareupProvider.loadPayments(str(job['job_id']), strCSVFilePath) log.writeInfo('[SQUARE UP] Transactions processed: ('+str(squareupProvider.getPaymentCount())+') | JobId: ('+str(job['job_id'])+') | BatchId: ('+str(job['batch_id'])+') | Date: ('+strDate(job['batch_key'])+').') except Exception, e: log.writeError(e)
def process(job, amazonProperties): amazonProvider = amazon.amazon(amazonProperties['AWS_ACCESS_KEY'], amazonProperties['AWS_SECRET_KEY']) amazonProvider.getPayments(dtDate(job['batch_key'])-timedelta(days=1), dtDate(job['batch_key'])) if amazonProvider.getTransactionCount() > 0: try: path = amazonProperties['datafile_path'] strCSVFilePath = path + "/amazon-payment-" + strDate(job['batch_key']) + ".csv" log.writeInfo(' [AMAZON] Writing '+strCSVFilePath+' to disk.') payment.write(amazonProvider, strCSVFilePath) payment.load(str(job['job_id']), strCSVFilePath) #logging.info('['+getTimeStamp()+'] [AMAZON] Transactions processed: ('+str(amazonProvider.getTransactionCount())+') | JobId: ('+job['job_id']+') | BatchId: ('+job['batch_id']+') | Date: ('+strDate+').') except Exception, e: log.writeError(e)
def getPaymentData(self, startDate): try: result = StringIO() c = pycurl.Curl() URL = 'https://connect.squareup.com/v1/me/payments?begin_time='+startDate+'T00:00:00Z&end_time='+startDate+'T23:59:59Z' c.setopt(pycurl.URL, URL) c.setopt(pycurl.HTTPHEADER, ['Authorization: Bearer '+self.getPersonalAccessToken()+'']) c.setopt(pycurl.WRITEFUNCTION, result.write) c.perform() resultDict = {} resultDict = result.getvalue() self._payments = json.loads(resultDict) except Exception, e: log.writeError(e)
def getPaymentData(self, startDate, endDate): try: payments = {} self.getEventData(startDate, endDate) for event in self.getEvents(): if 'fee' in event: self.getRsvpData(str(event['id'])) rsvps = self.getRsvps() for rsvp in rsvps: if rsvp['pay_status'] in ('paid','refunded'): payments[str(datetime.fromtimestamp(int(rsvp["mtime"])/1000).strftime('%Y%m%d%H%M%S'))] = {'member_id': rsvp["member"]["member_id"], 'member_name':rsvp["member"]["name"], 'event_id':rsvp["event"]["id"], 'event_name':rsvp["event"]["name"], 'date': datetime.fromtimestamp(int(rsvp["mtime"])/1000).strftime('%Y-%m-%d %H:%M:%S'), 'pay_status': rsvp["pay_status"], 'amount':event["fee"]["amount"]} self._payments = payments except Exception, e: log.writeError(e)
def process(job, meetupProperties): try: meetupProvider = meetup.meetup(meetupProperties['apiKey'], meetupProperties['groupName']) meetupProvider.getPaymentData(dtDate(job['batch_key']), dtDate(job['batch_key'])) if meetupProvider.getPaymentCount() > 0: strCSVFilePath = meetupProperties['datafile_path'] + '/meetup-payment-' + strDate(job['batch_key']) + '.csv' log.writeInfo(' [MEETUP] Writing '+strCSVFilePath+' to disk.') meetupProvider.writePayments(strCSVFilePath) meetupProvider.loadPayments(job['job_id'], strCSVFilePath) log.writeInfo(' [MEETUP] Transactions processed: ('+str(meetupProvider.getPaymentCount())+') | JobId: ('+job['job_id']+') | BatchId: ('+job['batch_id']+') | Date: ('+strDate(job['batch_key'])+').') else: skip(job) except Exception, e: log.writeError(e)
def writePayments(self, strCSVFilePath): if self.getPayments() == None: return None try: writer = csv.writer(open(strCSVFilePath, 'w'), quoting=csv.QUOTE_ALL) writer.writerow(["key", "datetime", "event_key", "event_name", "member_key", "member_name", "amount"]) payments = self.getPayments() for transaction in payments: row = list() row.append(str(transaction)+str(payments[transaction]['event_id'])+str(payments[transaction]['member_id'])) row.append(str(payments[transaction]['date'])) row.append(str(payments[transaction]['event_id'])) row.append(str(payments[transaction]['event_name'])) row.append(str(payments[transaction]['member_id'])) row.append(str(payments[transaction]['member_name'])) row.append(str(payments[transaction]['amount'])) writer.writerow(row) except Exception, e: log.writeError(e)
def get(amazonProvider): try: options = {} options['host'] = 'fps.amazonaws.com' boto_aws = FPSConnection(amazonProvider.getAccessKey(), amazonProvider.getSecretKey(), **options) options = {} options['StartDate'] = amazonProvider.getStartDate().strftime("%Y-%m-%d") options['EndDate'] = amazonProvider.getEndDate().strftime("%Y-%m-%d") obj = boto_aws.get_account_activity(**options) result = obj.GetAccountActivityResult amazonProvider._payments = result.Transaction return result.Transaction except Exception, e: log.writeError(e) return None
def loadPayments(self): try: payment.load(self) except Exception, e: log.writeError(e)
def getRsvpData(self, strEventId): try: rsvpURL='http://api.meetup.com/2/rsvps?key='+self.getApiKey()+'&event_id='+strEventId+'&fields=pay_status' self._rsvps = json.load(urllib2.urlopen(rsvpURL))['results'] except Exception, e: log.writeError(e)
def loadPayments(self, jobId, strCSVFilePath): try: result = db.execute(self.sqlLoadPayments(jobId, strCSVFilePath)) except Exception, e: log.writeError(e)